diff --git a/docs/api.md b/docs/api.md index 8bc172d..1f6a91e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -537,6 +537,7 @@ API는 이 필드 전송과 별개로 실제 DB 상태 검사·생성 차단을 "intention": "...", "userAnswer": "...", "modelAnswer": "...", + "followUpQuestion": "다른 대안과 비교한 근거는 무엇인가요?", "strengths": [], "improvements": [], "comment": "..." @@ -1002,3 +1003,17 @@ PDF 프로세스 간 복사 비용과 기존 그림 의미 분석의 중복 호 정상 문자로 잘못 매핑된 글자와 일부 작은 문구·읽기 순서 문제는 초기 탐지 규칙이 놓칠 수 있다. 제목·연락처만 남은 경우의 유용성도 이 평가에 포함한다. 정제 후 급감은 진단만 남기며 재호출하지 않는다. 실문서 평가 전에는 기본 비활성 상태를 유지한다. 자동 테스트 통과는 OCR 품질 검증을 대신하지 않는다. + +### 피드백 선택 필드와 기업 평가 복구 + +- solo/multi의 문항별 콜백은 선택 필드인 `followUpQuestion`을 제공한다. 실제 답변에 기반한 복습용 질문 한 문장이며 실제 면접에 삽입하지 않는다. +- 앞뒤 공백 제거 후 1~200 Unicode code point만 허용한다. 누락·공백·타입 오류·길이 초과는 null로 두고 채점은 유지한다. +- API 서버도 해당 필드의 저장·조회 매핑을 추가해야 하며 웹의 고정 안내 문구를 응답 필드에 연결해야 표시된다. +- 생성 문장의 Q 번호는 콜백 `feedbacks` 순서를 따른다. 확실한 내부 ID 참조만 교정하며 질문·답변 원문과 식별자는 바꾸지 않는다. +- SOLO/MULTI 채점과 기업 평가의 strict 도구 스키마에는 `minLength`, `maxLength`, `maxItems`를 보내지 않는다. + 길이·개수 상한은 필드 설명으로 안내하고 서버에서 검증한다. 예상 꼬리질문 1~200자, 기업 총평 1~300자, + 근거·제안 1~150자, 각 목록 최대 3개 정책은 유지한다. `question_ids.minItems=1`은 지원되는 제약이라 유지한다. + 지원 범위는 [Anthropic structured outputs 문서](https://platform.claude.com/docs/en/build-with-claude/structured-outputs#json-schema-limitations)를 따른다. +- 기업 평가는 strict 도구 제출을 사용하고 잘못된 제출만 최대 한 번 다시 생성한다. 두 호출은 하나의 `COMPANY_FIT_TIMEOUT_SECONDS` 예산을 공유한다. +- API 오류·토큰 잘림·시간 초과에는 추가 재시도하지 않는다. 기업 평가가 실패해도 일반 채점은 유지한다. +- `company_fit.*` 로그의 `job_id`로 요청을 구분한다. 완료 로그에는 적용 여부·시도·종료 사유·출력 토큰·경과 시간이 있고 원문은 기록하지 않는다. diff --git a/src/app/core/commands/dispatch_feedback_multi.py b/src/app/core/commands/dispatch_feedback_multi.py index 3f9e283..bb6ea5b 100644 --- a/src/app/core/commands/dispatch_feedback_multi.py +++ b/src/app/core/commands/dispatch_feedback_multi.py @@ -8,6 +8,7 @@ from app.core.common.company.fit.dto import CompanyFitPayload from app.core.common.company.fit.evaluate import CompanyFitEvaluation +from app.core.common.company.fit.logging_context import company_fit_job from app.core.common.feedback.concurrent import run_with_company_fit from app.core.common.feedback.dto import FeedbackCallbackFailure, FeedbackErrorDetail from app.core.common.feedback.multi.answer_grading import MultiAnswerGrading @@ -17,6 +18,8 @@ MultiFeedbackCallbackSuccess, MultiInterviewFeedbackResult, ) +from app.core.common.feedback.presentation import normalize_follow_up_question, normalize_generated_fields +from app.core.common.feedback.question_label import FEEDBACK_TEXT_FIELDS, OVERALL_TEXT_FIELDS, PERSONA_TEXT_FIELDS from app.core.common.feedback.scoring import ( AxisLevels, SessionScores, @@ -105,7 +108,8 @@ async def _build_payload(self, job_id: str, job_request: FeedbackMultiRequest) - ), ) fit = self._evaluate_company_fit(assembled, job_request) - raw_result, company_fit = await run_with_company_fit(grading, fit) + with company_fit_job(job_id): + raw_result, company_fit = await run_with_company_fit(grading, fit) levels = _question_levels(assembled, persona_by_question, raw_result) scores = _session_scores(levels, raw_result) result = self._build_result(job_request, assembled, persona_by_question, raw_result, levels, scores) @@ -157,7 +161,7 @@ async def _evaluate_company_fit( ) return await self._company_fit.execute(assembled.targets, job_request.company, tone) except Exception as exc: - logger.warning("feedback_multi.company_fit.failed", extra={"error_type": type(exc).__name__}) + logger.warning("feedback_multi.company_fit.failed error_type=%s", type(exc).__name__) return None def _build_result( @@ -169,8 +173,15 @@ def _build_result( levels: dict[str, AxisLevels], scores: SessionScores, ) -> MultiInterviewFeedbackResult: - graded: dict[str, Any] = raw_result["feedbacks"] - graded_personas: dict[str, Any] = raw_result["personas"] + question_ids = tuple(target.question_id for target in assembled.targets) + graded = { + qid: normalize_generated_fields(entry, (*FEEDBACK_TEXT_FIELDS, "follow_up_question"), question_ids) + for qid, entry in raw_result["feedbacks"].items() + } + graded_personas = { + pid: normalize_generated_fields(entry, PERSONA_TEXT_FIELDS, question_ids) + for pid, entry in raw_result["personas"].items() + } # 문항 순서는 LLM 응답 순서가 아니라 실제 면접 진행 순서를 따른다. # 질문 본문/의도/사용자 답변/면접관은 요청 body 값을 그대로 되돌려준다(LLM 생성분이 아니다). @@ -181,6 +192,9 @@ def _build_result( "question_content": target.content, "intention": target.intention, "user_answer": target.answer, + "follow_up_question": normalize_follow_up_question( + graded[target.question_id].get("follow_up_question") + ), "model_answer": graded[target.question_id].get("model_answer", ""), "strengths": graded[target.question_id].get("strengths", []), "improvements": graded[target.question_id].get("improvements", []), @@ -202,7 +216,7 @@ def _build_result( for persona in job_request.personas ] - graded_overall: dict[str, Any] = raw_result["overall"] + graded_overall = normalize_generated_fields(raw_result["overall"], OVERALL_TEXT_FIELDS, question_ids) # 3지표는 LLM 이 아니라 서버가 축 등급으로 계산한 값이다(scoring.py). overall: dict[str, Any] = { "total_score": scores.total_score, diff --git a/src/app/core/commands/dispatch_feedback_solo.py b/src/app/core/commands/dispatch_feedback_solo.py index 4e055eb..59e7bc0 100644 --- a/src/app/core/commands/dispatch_feedback_solo.py +++ b/src/app/core/commands/dispatch_feedback_solo.py @@ -8,8 +8,11 @@ from app.core.common.company.fit.dto import CompanyFitPayload from app.core.common.company.fit.evaluate import CompanyFitEvaluation +from app.core.common.company.fit.logging_context import company_fit_job from app.core.common.feedback.concurrent import run_with_company_fit from app.core.common.feedback.dto import FeedbackCallbackFailure, FeedbackErrorDetail +from app.core.common.feedback.presentation import normalize_follow_up_question, normalize_generated_fields +from app.core.common.feedback.question_label import FEEDBACK_TEXT_FIELDS, OVERALL_TEXT_FIELDS from app.core.common.feedback.scoring import SessionScores, summarize_session from app.core.common.feedback.solo.answer_assembly import AnswerAssembly from app.core.common.feedback.solo.answer_grading import AnswerGrading @@ -83,7 +86,8 @@ async def _build_payload(self, job_id: str, job_request: FeedbackSoloRequest) -> ), ) fit = self._evaluate_company_fit(assembled, job_request) - raw_result, company_fit = await run_with_company_fit(grading, fit) + with company_fit_job(job_id): + raw_result, company_fit = await run_with_company_fit(grading, fit) scores = _session_scores(assembled, raw_result) result = self._build_result(assembled, raw_result, scores) result.company_fit = company_fit @@ -132,7 +136,7 @@ async def _evaluate_company_fit( tone = job_request.persona_tone return await self._company_fit.execute(assembled.targets, job_request.company, tone) except Exception as exc: - logger.warning("feedback_solo.company_fit.failed", extra={"error_type": type(exc).__name__}) + logger.warning("feedback_solo.company_fit.failed error_type=%s", type(exc).__name__) return None def _build_result( @@ -141,7 +145,11 @@ def _build_result( raw_result: dict[str, Any], scores: SessionScores, ) -> InterviewFeedbackResult: - graded: dict[str, Any] = raw_result["feedbacks"] + question_ids = tuple(target.question_id for target in assembled.targets) + graded = { + qid: normalize_generated_fields(entry, (*FEEDBACK_TEXT_FIELDS, "follow_up_question"), question_ids) + for qid, entry in raw_result["feedbacks"].items() + } # 문항 순서는 LLM 응답 순서가 아니라 실제 면접 진행 순서를 따른다. # 질문 본문/의도/사용자 답변은 요청 body 값을 그대로 되돌려준다(LLM 생성분이 아니다). @@ -151,6 +159,9 @@ def _build_result( "question_content": target.content, "intention": target.intention, "user_answer": target.answer, + "follow_up_question": normalize_follow_up_question( + graded[target.question_id].get("follow_up_question") + ), "model_answer": graded[target.question_id].get("model_answer", ""), "strengths": graded[target.question_id].get("strengths", []), "improvements": graded[target.question_id].get("improvements", []), @@ -159,7 +170,7 @@ def _build_result( for target in assembled.targets ] - graded_overall: dict[str, Any] = raw_result["overall"] + graded_overall = normalize_generated_fields(raw_result["overall"], OVERALL_TEXT_FIELDS, question_ids) # 3지표는 LLM 이 아니라 서버가 축 등급으로 계산한 값이다(scoring.py). overall: dict[str, Any] = { "total_score": scores.total_score, diff --git a/src/app/core/common/company/fit/evaluate.py b/src/app/core/common/company/fit/evaluate.py index 259b48d..3dd4598 100644 --- a/src/app/core/common/company/fit/evaluate.py +++ b/src/app/core/common/company/fit/evaluate.py @@ -16,14 +16,16 @@ CompanyFitItem, CompanyFitPayload, ) +from app.core.common.company.fit.logging_context import CompanyFitLogger from app.core.common.company.fit.prompt import SYSTEM_PROMPT, build_fit_user_message from app.core.common.company.fit.tools import SUBMIT_COMPANY_FIT_TOOL, TOOL_NAME +from app.core.common.feedback.presentation import normalize_generated_fields from app.core.common.feedback.question_label import warn_out_of_range_labels from app.core.common.feedback.solo.dto import GradingTarget -from app.core.common.interview_qa.ports.anthropic_text_client import AnthropicTextClient +from app.core.common.interview_qa.ports.anthropic_text_client import AnthropicTextClient, AnthropicTextClientError from app.core.common.tool_use import extract_tool_input -logger = logging.getLogger(__name__) +logger = CompanyFitLogger(logging.getLogger(__name__), {}) class CompanyFitEvaluation: @@ -52,58 +54,80 @@ async def execute( persona_tone: str | None, ) -> CompanyFitPayload | None: started = time.monotonic() + message = build_fit_user_message(targets, company, persona_tone, self._answer_max_chars) + question_ids = tuple(target.question_id for target in targets) try: - # SDK 내부 대기·재시도를 포함해 fit 시작부터 전체 논리 호출을 제한한다. + # SDK 내부 재시도 및 형식 보정 1회를 포함한 전체 예산이다. async with asyncio.timeout(self._timeout_seconds): - response = await self._client.call( - model=self._model, - system=SYSTEM_PROMPT, - messages=[ - { - "role": "user", - "content": build_fit_user_message(targets, company, persona_tone, self._answer_max_chars), - } - ], - tools=[SUBMIT_COMPANY_FIT_TOOL], - tool_choice={"type": "tool", "name": TOOL_NAME}, - max_tokens=self._max_tokens, - temperature=self._temperature, - ) + for attempt in (1, 2): + response = await self._client.call( + model=self._model, + system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": message}], + tools=[SUBMIT_COMPANY_FIT_TOOL], + tool_choice={"type": "tool", "name": TOOL_NAME}, + max_tokens=self._max_tokens, + temperature=self._temperature, + ) + tool_blocks = [block for block in response.content_blocks if block.get("type") == "tool_use"] + raw = extract_tool_input(response.content_blocks, TOOL_NAME) + result = None + if raw is None or response.stop_reason != "tool_use" or len(tool_blocks) != 1: + logger.warning( + "company_fit.submission_missing_or_truncated stop_reason=%s attempt=%d", + response.stop_reason, + attempt, + ) + else: + result = _parse_submission(raw, company, set(question_ids)) + if result is not None: + _normalize_references(result, question_ids) + logger.info( + "company_fit.completed applied=%s attempt=%d stop_reason=%s output_tokens=%d elapsed_ms=%.0f", + result is not None, + attempt, + response.stop_reason, + response.output_tokens, + (time.monotonic() - started) * 1000, + extra={ + "applied": result is not None, + "matched_count": len(result.matched_focus) if result else 0, + "gap_count": len(result.gaps) if result else 0, + "input_tokens": response.input_tokens, + "output_tokens": response.output_tokens, + "attempt": attempt, + "elapsed_ms": (time.monotonic() - started) * 1000, + }, + ) + if result is not None or response.stop_reason == "max_tokens": + return result + if attempt == 1: + logger.warning("company_fit.retry reason=invalid_submission attempt=1") + message += ( + "\n직전 응답이 제출 형식을 충족하지 못했습니다. " + "지정된 도구로 필수 필드와 길이를 지키고 제공된 fact_id·question_id만 사용해 제출하세요." + ) except TimeoutError: - logger.warning("company_fit.timeout", extra={"elapsed_ms": (time.monotonic() - started) * 1000}) - return None - tool_blocks = [block for block in response.content_blocks if block.get("type") == "tool_use"] - raw = extract_tool_input(response.content_blocks, TOOL_NAME) - result = None - if raw is None or response.stop_reason != "tool_use" or len(tool_blocks) != 1: - logger.warning("company_fit.submission_missing_or_truncated") - else: - result = _parse_submission(raw, company, {target.question_id for target in targets}) - if result is not None: - # 문장에 블록 번호(Q1…) 대신 question_id 를 쓴 흔적을 찾는다. 로그만 남기고 결과는 그대로 쓴다. - # 블록 번호는 targets 순서로 매기므로 대상 수도 targets 기준이다. - warn_out_of_range_labels( - logger, - "company_fit.question_id_label", - [ - result.comment, - *(item.evidence for item in result.matched_focus), - *(gap.suggestion for gap in result.gaps), - ], - len(targets), - ) - logger.info( - "company_fit.completed", - extra={ - "applied": result is not None, - "matched_count": len(result.matched_focus) if result else 0, - "gap_count": len(result.gaps) if result else 0, - "input_tokens": response.input_tokens, - "output_tokens": response.output_tokens, - "elapsed_ms": (time.monotonic() - started) * 1000, - }, - ) - return result + logger.warning("company_fit.timeout elapsed_ms=%.0f", (time.monotonic() - started) * 1000) + except AnthropicTextClientError as exc: + logger.warning("company_fit.api_failed reason=%s", exc.kind) + except Exception as exc: + logger.warning("company_fit.failed error_type=%s", type(exc).__name__) + return None + + +def _normalize_references(result: CompanyFitPayload, question_ids: tuple[str, ...]) -> None: + warn_out_of_range_labels( + logger, + "company_fit.question_id_label", + [result.comment, *(item.evidence for item in result.matched_focus), *(gap.suggestion for gap in result.gaps)], + len(question_ids), + ) + result.comment = normalize_generated_fields({"comment": result.comment}, ("comment",), question_ids)["comment"] + for item in result.matched_focus: + item.evidence = normalize_generated_fields({"text": item.evidence}, ("text",), question_ids)["text"] + for gap in result.gaps: + gap.suggestion = normalize_generated_fields({"text": gap.suggestion}, ("text",), question_ids)["text"] def _parse_submission( diff --git a/src/app/core/common/company/fit/logging_context.py b/src/app/core/common/company/fit/logging_context.py new file mode 100644 index 0000000..a2e9d24 --- /dev/null +++ b/src/app/core/common/company/fit/logging_context.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import logging +from collections.abc import Iterator, MutableMapping +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any + +_job_id: ContextVar[str | None] = ContextVar("company_fit_job_id", default=None) + + +@contextmanager +def company_fit_job(job_id: str) -> Iterator[None]: + token = _job_id.set(job_id) + try: + yield + finally: + _job_id.reset(token) + + +class CompanyFitLogger(logging.LoggerAdapter[logging.Logger]): + def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> tuple[Any, MutableMapping[str, Any]]: + job_id = _job_id.get() + if job_id is not None: + # job_id는 HTTP 진입점에서 발급한 UUID다. 모델 응답·사용자 입력은 넣지 않는다. + msg = f"{msg} job_id={job_id}" + kwargs["extra"] = {**kwargs.get("extra", {}), "job_id": job_id} + return msg, kwargs diff --git a/src/app/core/common/company/fit/tools.py b/src/app/core/common/company/fit/tools.py index b82dfbc..68a0549 100644 --- a/src/app/core/common/company/fit/tools.py +++ b/src/app/core/common/company/fit/tools.py @@ -8,12 +8,12 @@ def _items_schema(text_field: str) -> dict[str, Any]: return { "type": "array", - "maxItems": FIT_ITEMS_MAX, + "description": f"최대 {FIT_ITEMS_MAX}개. 답변 근거가 없으면 빈 배열.", "items": { "type": "object", "properties": { "fact_id": {"type": "string"}, - text_field: {"type": "string", "minLength": 1, "maxLength": FIT_ITEM_TEXT_MAX}, + text_field: {"type": "string", "description": f"답변에 근거한 한국어 문장. 1~{FIT_ITEM_TEXT_MAX}자."}, "question_ids": {"type": "array", "minItems": 1, "items": {"type": "string"}}, }, "required": ["fact_id", text_field, "question_ids"], @@ -24,11 +24,13 @@ def _items_schema(text_field: str) -> dict[str, Any]: SUBMIT_COMPANY_FIT_TOOL: dict[str, Any] = { "name": TOOL_NAME, + "strict": True, "description": "답변 근거에 기반한 기업 관점 코멘트를 제출한다 점수나 등급을 포함하지 않는다", "input_schema": { "type": "object", "properties": { - "comment": {"type": "string", "minLength": 1, "maxLength": FIT_COMMENT_MAX}, + # strict가 지원하지 않는 길이·개수 상한은 설명에만 두고 파서가 검증한다. + "comment": {"type": "string", "description": f"기업 관점 총평. 1~2문장 한국어, 1~{FIT_COMMENT_MAX}자."}, "matched_focus": _items_schema("evidence"), "gaps": _items_schema("suggestion"), }, diff --git a/src/app/core/common/feedback/multi/prompt.py b/src/app/core/common/feedback/multi/prompt.py index fcb534d..98eab92 100644 --- a/src/app/core/common/feedback/multi/prompt.py +++ b/src/app/core/common/feedback/multi/prompt.py @@ -4,6 +4,7 @@ from collections.abc import Mapping, Sequence from app.core.common.feedback.multi.dto import FeedbackPersona +from app.core.common.feedback.presentation import FOLLOW_UP_QUESTION_GUIDANCE from app.core.common.feedback.rubric import AXIS_LEVEL_CRITERIA, general_axis_level_criteria from app.core.common.feedback.solo.dto import AssembledSession, GradingTarget from app.core.common.job_interview import interviewer_title, specialist_job @@ -74,6 +75,7 @@ - comment는 해당 면접관 시점의 한 문장짜리 총평이다. [문항별 피드백 feedbacks] + {follow_up_guidance} - model_answer: 질문 의도를 충족하는 40~100자의 짧은 예시 답안. 사용자에게 보여주는 예시이지 채점 기준이 아니다. 해당 면접관의 직책 관점에서 쓰고, 제공되지 않은 사실을 지어내지 마라. @@ -114,7 +116,7 @@ 점검 과정은 출력하지 말고 반드시 submit_multi_feedback 도구로 결과를 제출하라. """ -).format(axis_criteria=AXIS_LEVEL_CRITERIA) +).format(axis_criteria=AXIS_LEVEL_CRITERIA, follow_up_guidance=FOLLOW_UP_QUESTION_GUIDANCE) def build_grading_user_message( diff --git a/src/app/core/common/feedback/multi/tools.py b/src/app/core/common/feedback/multi/tools.py index 149da78..7056464 100644 --- a/src/app/core/common/feedback/multi/tools.py +++ b/src/app/core/common/feedback/multi/tools.py @@ -4,6 +4,7 @@ from typing import Any from app.core.common.feedback.depth import DEPTH_EVIDENCE_SCHEMA +from app.core.common.feedback.presentation import FOLLOW_UP_QUESTION_SCHEMA from app.core.common.job_interview import specialist_job # solo/tools.py 를 그대로 재사용하지 않고 따로 두는 이유는 설명문이 달라야 하기 때문이다. @@ -45,6 +46,7 @@ "type": "string", "description": "채점 대상으로 전달받은 question_id 를 그대로 사용한다.", }, + "follow_up_question": FOLLOW_UP_QUESTION_SCHEMA, "model_answer": { "type": "string", "description": ( @@ -69,7 +71,16 @@ "depth_evidence": DEPTH_EVIDENCE_SCHEMA, "scores": _AXIS_SCORES_SCHEMA, }, - "required": ["question_id", "model_answer", "strengths", "improvements", "comment", "depth_evidence", "scores"], + "required": [ + "question_id", + "model_answer", + "strengths", + "improvements", + "comment", + "depth_evidence", + "scores", + "follow_up_question", + ], } diff --git a/src/app/core/common/feedback/presentation.py b/src/app/core/common/feedback/presentation.py new file mode 100644 index 0000000..8e3995e --- /dev/null +++ b/src/app/core/common/feedback/presentation.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import inspect +import logging +import re +from collections.abc import Mapping, Sequence +from typing import Any + +logger = logging.getLogger(__name__) + +FOLLOW_UP_QUESTION_MAX = 200 +FOLLOW_UP_QUESTION_SCHEMA: dict[str, Any] = { + "type": "string", + # strict 도구는 문자열 길이 제약을 지원하지 않는다. 길이는 서버에서 검증한다. + "description": ( + "현재 질문과 실제 답변의 근거·예외·트레이드오프를 더 확인하는 예상 꼬리질문 한 문장. " + f"1~{FOLLOW_UP_QUESTION_MAX}자 한국어로 쓴다." + ), +} +FOLLOW_UP_QUESTION_GUIDANCE = inspect.cleandoc( + """ + - follow_up_question: 실제 답변의 근거·예외·트레이드오프 중 하나를 더 확인하는 예상 꼬리질문 한 문장. + 현재 질문을 단순 반복하지 말고, 답변에 없는 경험·수치·구현을 했다고 전제하지 마라. + 질문만 200자 이내 한국어로 쓰고 점수·모범답안·목록 번호는 쓰지 마라. + 이 질문은 복습용이며 실제 면접에 삽입하거나 이미 출제된 질문이라고 주장하지 마라. +""" +) +_REFERENCE = re.compile(r"(?Q|문항[ \t]*)(?P[0-9]+)(?![A-Za-z0-9_])") + + +def normalize_follow_up_question(value: object) -> str | None: + if value is None: + return None + if isinstance(value, str): + text = value.strip() + if text and len(text) <= FOLLOW_UP_QUESTION_MAX: + return text + # 문장이나 잘못된 객체를 로그에 남기지 않는다. 선택 항목 실패는 채점 전체를 막지 않는다. + logger.warning("feedback.follow_up_question.invalid") + return None + + +def normalize_generated_fields( + entry: Mapping[str, Any], fields: Sequence[str], question_ids: Sequence[str] +) -> dict[str, Any]: + """확실한 내부 ID 참조만 표시 번호로 바꾼다. 원문과 식별자 필드는 호출자가 제외한다.""" + positions = {qid: str(index) for index, qid in enumerate(question_ids, 1)} + ambiguous = {str(index) for index in range(1, len(question_ids) + 1)} + + def replace(match: re.Match[str]) -> str: + qid = match["id"] + # 표시 범위와 겹치는 값, 알 수 없는 값, 중복 ID는 의미를 추측하지 않는다. + if qid in ambiguous or qid not in positions or question_ids.count(qid) != 1: + return match[0] + return match["prefix"] + positions[qid] + + def normalize(value: Any) -> Any: + if isinstance(value, str): + return _REFERENCE.sub(replace, value) + if isinstance(value, list): + return [_REFERENCE.sub(replace, item) if isinstance(item, str) else item for item in value] + return value + + return {**entry, **{field: normalize(entry[field]) for field in fields if field in entry}} diff --git a/src/app/core/common/feedback/question_label.py b/src/app/core/common/feedback/question_label.py index cce3d5f..16f2853 100644 --- a/src/app/core/common/feedback/question_label.py +++ b/src/app/core/common/feedback/question_label.py @@ -32,7 +32,7 @@ def out_of_range_labels(texts: Iterable[object], target_count: int) -> list[str] def warn_out_of_range_labels( - log: logging.Logger, + log: logging.Logger | logging.LoggerAdapter[logging.Logger], event: str, texts: Iterable[object], target_count: int, diff --git a/src/app/core/common/feedback/solo/dto.py b/src/app/core/common/feedback/solo/dto.py index d103ca3..dccbbb8 100644 --- a/src/app/core/common/feedback/solo/dto.py +++ b/src/app/core/common/feedback/solo/dto.py @@ -8,6 +8,7 @@ from app.core.common.company.dto import CompanyContext from app.core.common.company.fit.dto import CompanyFitPayload from app.core.common.feedback.dto import CamelModel, ScoreBreakdownPayload +from app.core.common.feedback.presentation import FOLLOW_UP_QUESTION_MAX # 소켓 서버(Java) 의 QuestionType enum 과 같은 값. Jackson 이 enum 이름을 그대로 직렬화한다. # ORIGINAL: 최초 질문, FOLLOW: 꼬리 질문(parent_id 로 부모를 가리킨다). @@ -86,6 +87,7 @@ class AnswerFeedback(CamelModel): user_answer: str # 채점 기준이 아니라 사용자에게 보여주는 예시 답안. 40~100자. model_answer: str + follow_up_question: str | None = Field(default=None, min_length=1, max_length=FOLLOW_UP_QUESTION_MAX) strengths: list[str] improvements: list[str] comment: str # 한 줄 총평. diff --git a/src/app/core/common/feedback/solo/prompt.py b/src/app/core/common/feedback/solo/prompt.py index 7f2a00d..b007271 100644 --- a/src/app/core/common/feedback/solo/prompt.py +++ b/src/app/core/common/feedback/solo/prompt.py @@ -2,6 +2,7 @@ import inspect +from app.core.common.feedback.presentation import FOLLOW_UP_QUESTION_GUIDANCE from app.core.common.feedback.rubric import AXIS_LEVEL_CRITERIA, general_axis_level_criteria from app.core.common.feedback.solo.dto import AssembledSession, GradingTarget from app.core.common.job_interview import interviewer_title, specialist_job @@ -40,6 +41,7 @@ {axis_criteria} [문항별 피드백] + {follow_up_guidance} - model_answer: 질문 의도를 충족하는 40~100자의 짧은 예시 답안. 사용자에게 '이렇게 답할 수 있다'고 보여주는 예시이지 채점 기준이 아니다. 제공되지 않은 프로젝트 사실이나 구현을 지어내지 마라. @@ -80,7 +82,7 @@ 점검 과정은 출력하지 말고 반드시 submit_feedback 도구로 결과를 제출하라. """ -).format(axis_criteria=AXIS_LEVEL_CRITERIA) +).format(axis_criteria=AXIS_LEVEL_CRITERIA, follow_up_guidance=FOLLOW_UP_QUESTION_GUIDANCE) def build_grading_user_message( diff --git a/src/app/core/common/feedback/solo/tools.py b/src/app/core/common/feedback/solo/tools.py index 2533304..b548027 100644 --- a/src/app/core/common/feedback/solo/tools.py +++ b/src/app/core/common/feedback/solo/tools.py @@ -4,6 +4,7 @@ from typing import Any from app.core.common.feedback.depth import DEPTH_EVIDENCE_SCHEMA +from app.core.common.feedback.presentation import FOLLOW_UP_QUESTION_SCHEMA from app.core.common.job_interview import specialist_job # 필드명을 snake_case 로 두는 이유: 파싱 결과를 그대로 InterviewFeedbackResult 로 넘기기 위해서다. @@ -44,6 +45,7 @@ "type": "string", "description": "채점 대상으로 전달받은 question_id 를 그대로 사용한다.", }, + "follow_up_question": FOLLOW_UP_QUESTION_SCHEMA, "model_answer": { "type": "string", "description": ( @@ -68,7 +70,16 @@ "depth_evidence": DEPTH_EVIDENCE_SCHEMA, "scores": _AXIS_SCORES_SCHEMA, }, - "required": ["question_id", "model_answer", "strengths", "improvements", "comment", "depth_evidence", "scores"], + "required": [ + "question_id", + "model_answer", + "strengths", + "improvements", + "comment", + "depth_evidence", + "scores", + "follow_up_question", + ], } diff --git a/tests/core/commands/test_feedback_company_fit.py b/tests/core/commands/test_feedback_company_fit.py index da02b1e..55ffb0a 100644 --- a/tests/core/commands/test_feedback_company_fit.py +++ b/tests/core/commands/test_feedback_company_fit.py @@ -39,7 +39,7 @@ async def test_company_fit_is_optional_and_cannot_change_grading_input_or_scores if fit_state == "absent": assert len(llm.calls) == 1 else: - assert len(llm.calls) == 2 + assert len(llm.calls) == (3 if fit_state == "invalid" else 2) fit = next(call for call in llm.calls if call["tool_choice"].get("name") == "submit_company_fit") assert fit["temperature"] == 0.2 assert fit["max_tokens"] == 1500 diff --git a/tests/core/commands/test_feedback_presentation.py b/tests/core/commands/test_feedback_presentation.py new file mode 100644 index 0000000..2b4611c --- /dev/null +++ b/tests/core/commands/test_feedback_presentation.py @@ -0,0 +1,51 @@ +from typing import Any + +import pytest + +from tests.company_helpers import company_http +from tests.multi_helpers import feedback_body + + +@pytest.mark.parametrize("mode", ["solo", "multi"]) +@pytest.mark.parametrize("follow_up", [None, "다른 방법은 무엇인가요?", " \n ", "가" * 201, {"bad": 1}]) +async def test_follow_up_survives_callback_and_invalid_optional_value_does_not_break_grading( + mode: str, follow_up: Any +) -> None: + async with company_http() as (client, webhook, llm): + tool = "submit_feedback" if mode == "solo" else "submit_multi_feedback" + llm.outputs[tool]["feedbacks"][0]["follow_up_question"] = follow_up + response = await client.post(f"/feedback/{mode}", json=feedback_body()) + assert response.status_code == 202 + payload = webhook.payloads[0] + assert payload["status"] == "succeeded" + expected = follow_up if follow_up == "다른 방법은 무엇인가요?" else None + assert payload["result"]["feedbacks"][0]["followUpQuestion"] == expected + assert "follow_up_question" not in payload["result"]["feedbacks"][0] + + +@pytest.mark.parametrize("mode", ["solo", "multi"]) +async def test_internal_ids_are_corrected_only_in_generated_callback_text(mode: str) -> None: + body = feedback_body() + body["questions"][0]["questionId"] = "62" + body["questions"][0]["content"] = "원래 질문 Q62" + body["answers"][0]["questionId"] = "62" + body["answers"][0]["content"] += " Q62 원문은 유지합니다." + async with company_http() as (client, webhook, llm): + tool = "submit_feedback" if mode == "solo" else "submit_multi_feedback" + output = llm.outputs[tool] + output["feedbacks"][0].update(question_id="62", comment="Q62 근거", follow_up_question="문항 62 대안은?") + output["overall"]["summary"] = "Q62에서 판단했습니다" + if mode == "multi": + output["personas"][0]["comment"] = "Q62 보완" + response = await client.post(f"/feedback/{mode}", json=body) + assert response.status_code == 202 + result = webhook.payloads[0]["result"] + assert result["overall"]["summary"] == "Q1에서 판단했습니다" + item = result["feedbacks"][0] + assert item["questionId"] == "62" + assert item["comment"] == "Q1 근거" + assert item["followUpQuestion"] == "문항 1 대안은?" + assert item["questionContent"] == "원래 질문 Q62" + assert item["userAnswer"] == body["answers"][0]["content"] + if mode == "multi": + assert result["personas"][-1]["comment"] == "Q1 보완" diff --git a/tests/core/common/company/test_fit_contract.py b/tests/core/common/company/test_fit_contract.py index 721fbba..9d84c7a 100644 --- a/tests/core/common/company/test_fit_contract.py +++ b/tests/core/common/company/test_fit_contract.py @@ -93,7 +93,7 @@ async def test_completion_logs_usage_and_counts_without_raw_text(caplog: pytest. with caplog.at_level(logging.INFO): result = await CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000).execute((target,), company, None) assert result is not None - (record,) = [record for record in caplog.records if record.msg == "company_fit.completed"] + (record,) = [record for record in caplog.records if record.getMessage().startswith("company_fit.completed")] assert record.__dict__["applied"] is True assert record.__dict__["matched_count"] == record.__dict__["gap_count"] == 1 assert record.__dict__["input_tokens"] >= 0 diff --git a/tests/core/common/company/test_fit_recovery.py b/tests/core/common/company/test_fit_recovery.py new file mode 100644 index 0000000..0456b5c --- /dev/null +++ b/tests/core/common/company/test_fit_recovery.py @@ -0,0 +1,130 @@ +import asyncio +import logging +from typing import Any + +import pytest + +from app.core.common.company.fit.evaluate import CompanyFitEvaluation +from app.core.common.company.fit.logging_context import company_fit_job +from app.core.common.feedback.solo.dto import GradingTarget +from app.core.common.interview_qa.ports.anthropic_text_client import AnthropicCallResult, AnthropicTextClientError +from tests.company_helpers import RecordingCompanyClient, company_context, fit_output + + +def target(question_id: str = "q-1") -> GradingTarget: + return GradingTarget( + question_id=question_id, type="ORIGINAL", intention="선택 근거", content="질문", answer="비공개 답변" + ) + + +def response(raw: dict[str, Any], stop: str = "tool_use") -> AnthropicCallResult: + return AnthropicCallResult( + content_blocks=[{"type": "tool_use", "name": "submit_company_fit", "input": raw}], + input_tokens=10, + output_tokens=20, + stop_reason=stop, + ) + + +async def test_invalid_submission_is_retried_once_and_recovers(monkeypatch: pytest.MonkeyPatch) -> None: + client = RecordingCompanyClient() + calls: list[dict[str, Any]] = [] + + async def call(**kwargs: Any) -> AnthropicCallResult: + calls.append(kwargs) + raw = fit_output() + if len(calls) == 1: + raw["comment"] = "" + return response(raw) + + monkeypatch.setattr(client, "call", call) + result = await CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000).execute((target(),), company_context(), None) + assert result is not None + assert len(calls) == 2 + assert calls[0]["tools"][0]["strict"] is True + assert "직전 응답" in calls[1]["messages"][0]["content"] + + +@pytest.mark.parametrize("failure", ["truncated", "api"]) +async def test_truncated_or_api_error_does_not_trigger_application_retry( + failure: str, monkeypatch: pytest.MonkeyPatch +) -> None: + client = RecordingCompanyClient() + calls = 0 + + async def call(**kwargs: Any) -> AnthropicCallResult: + nonlocal calls + calls += 1 + if failure == "api": + raise AnthropicTextClientError("비공개 오류", kind="transient") + return response(fit_output(), "max_tokens") + + monkeypatch.setattr(client, "call", call) + assert ( + await CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000).execute((target(),), company_context(), None) + is None + ) + assert calls == 1 + + +async def test_retry_shares_deadline_and_cleans_up_call(monkeypatch: pytest.MonkeyPatch) -> None: + client = RecordingCompanyClient() + calls, cancelled = 0, asyncio.Event() + + async def call(**kwargs: Any) -> AnthropicCallResult: + nonlocal calls + calls += 1 + if calls == 1: + raw = fit_output() + raw["comment"] = "" + return response(raw) + try: + await asyncio.Future[None]() + finally: + cancelled.set() + raise AssertionError("unreachable") + + monkeypatch.setattr(client, "call", call) + evaluator = CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000, timeout_seconds=0.02) + assert await asyncio.wait_for(evaluator.execute((target(),), company_context(), None), 1) is None + assert calls == 2 + assert cancelled.is_set() + + +async def test_company_references_keep_fact_and_question_ids(monkeypatch: pytest.MonkeyPatch) -> None: + client = RecordingCompanyClient() + raw = fit_output() + raw["comment"] = "Q62 답변의 근거" + for field in ("matched_focus", "gaps"): + raw[field][0]["question_ids"] = ["62"] + raw["matched_focus"][0]["evidence"] = "문항 62 근거" + + async def call(**kwargs: Any) -> AnthropicCallResult: + return response(raw) + + monkeypatch.setattr(client, "call", call) + result = await CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000).execute( + (target("62"),), company_context(), None + ) + assert result is not None + assert result.comment == "Q1 답변의 근거" + assert result.matched_focus[0].evidence == "문항 1 근거" + assert result.matched_focus[0].fact_id == "t1" + assert result.matched_focus[0].question_ids == ["62"] + + +async def test_concurrent_logs_keep_job_context_and_do_not_leak_text(caplog: pytest.LogCaptureFixture) -> None: + client = RecordingCompanyClient() + + async def run(job_id: str) -> None: + with company_fit_job(job_id): + await asyncio.sleep(0) + await CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000).execute((target(),), company_context(), None) + + with caplog.at_level(logging.INFO): + await asyncio.gather(run("job-a"), run("job-b")) + await CompanyFitEvaluation(client, "stub", 1500, 0.2, 3000).execute((target(),), company_context(), None) + records = [record for record in caplog.records if record.getMessage().startswith("company_fit.completed")] + assert [getattr(record, "job_id", None) for record in records] == ["job-a", "job-b", None] + assert "job_id=job-a" in records[0].getMessage() + assert "비공개 답변" not in caplog.text diff --git a/tests/core/common/company/test_fit_server_limits.py b/tests/core/common/company/test_fit_server_limits.py new file mode 100644 index 0000000..260258b --- /dev/null +++ b/tests/core/common/company/test_fit_server_limits.py @@ -0,0 +1,52 @@ +import pytest + +from app.core.common.company.dto import CompanyFact +from app.core.common.company.fit.dto import FIT_COMMENT_MAX, FIT_ITEM_TEXT_MAX, FIT_ITEMS_MAX +from app.core.common.company.fit.evaluate import _parse_submission +from tests.company_helpers import company_context, fit_output + + +@pytest.mark.parametrize("field", ["matched_focus", "gaps"]) +def test_server_keeps_only_first_three_valid_distinct_facts(field: str) -> None: + company = company_context() + company = company.model_copy( + update={ + "facts": ( + *company.facts, + CompanyFact( + fact_id="fourth", + kind="BUSINESS", + title="추가 서비스", + content="네 번째 자료", + ), + ) + } + ) + raw = fit_output() + text_key = "evidence" if field == "matched_focus" else "suggestion" + raw[field] = [{"fact_id": fact.prompt_id, text_key: "답변 근거", "question_ids": ["q-1"]} for fact in company.facts] + result = _parse_submission(raw, company, {"q-1"}) + assert result is not None + items = result.matched_focus if field == "matched_focus" else result.gaps + assert len(items) == FIT_ITEMS_MAX + assert [item.fact_id for item in items] == [fact.fact_id for fact in company.facts[:FIT_ITEMS_MAX]] + + +@pytest.mark.parametrize("length", [FIT_COMMENT_MAX, FIT_COMMENT_MAX + 1]) +def test_comment_length_is_still_checked_by_server(length: int) -> None: + raw = fit_output() + raw["comment"] = "가" * length + result = _parse_submission(raw, company_context(), {"q-1"}) + assert (result is not None) is (length <= FIT_COMMENT_MAX) + + +@pytest.mark.parametrize("field", ["matched_focus", "gaps"]) +@pytest.mark.parametrize("length", [FIT_ITEM_TEXT_MAX, FIT_ITEM_TEXT_MAX + 1]) +def test_item_length_is_still_checked_by_server(field: str, length: int) -> None: + raw = fit_output() + text_key = "evidence" if field == "matched_focus" else "suggestion" + raw[field][0][text_key] = "가" * length + result = _parse_submission(raw, company_context(), {"q-1"}) + assert result is not None + items = result.matched_focus if field == "matched_focus" else result.gaps + assert len(items) == (1 if length <= FIT_ITEM_TEXT_MAX else 0) diff --git a/tests/core/common/feedback/test_presentation.py b/tests/core/common/feedback/test_presentation.py new file mode 100644 index 0000000..a86d876 --- /dev/null +++ b/tests/core/common/feedback/test_presentation.py @@ -0,0 +1,54 @@ +from copy import deepcopy + +import pytest + +from app.core.common.feedback.presentation import normalize_follow_up_question, normalize_generated_fields + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, None), + ("", None), + (" \n", None), + (3, None), + ({"private": "text"}, None), + ("가" * 201, None), + (" 질문? \n", "질문?"), + ("가" * 200, "가" * 200), + ("😀" * 200, "😀" * 200), + ], +) +def test_optional_follow_up_is_bounded_without_truncation(value: object, expected: str | None) -> None: + assert normalize_follow_up_question(value) == expected + + +def test_references_use_callback_order_without_touching_other_fields() -> None: + entry = { + "question_id": "62", + "user_answer": "Q62 답변", + "summary": "Q63과 문항 62를 비교", + "strengths": ["Q62 근거", "SQ62와 Q63a는 제품명"], + "depth_evidence": {"reason": "Q62 원문 인용"}, + } + original = deepcopy(entry) + result = normalize_generated_fields(entry, ("summary", "strengths"), ("63", "62")) + assert result["summary"] == "Q1과 문항 2를 비교" + assert result["strengths"] == ["Q2 근거", "SQ62와 Q63a는 제품명"] + assert result["question_id"] == "62" + assert result["user_answer"] == "Q62 답변" + assert result["depth_evidence"] == original["depth_evidence"] + assert entry == original + + +def test_ambiguous_unknown_and_duplicate_references_are_preserved() -> None: + fields = ("comment",) + entry = {"comment": "Q1 Q2 Q62 Q999"} + assert normalize_generated_fields(entry, fields, ("2", "62"))["comment"] == "Q1 Q2 Q2 Q999" + assert normalize_generated_fields(entry, fields, ("62", "62"))["comment"] == entry["comment"] + + +def test_opaque_follow_ids_do_not_break_reference_mapping() -> None: + entry = {"comment": "Q62와 Q1", "improvements": ["문항 66 보완", None]} + result = normalize_generated_fields(entry, ("comment", "improvements"), ("62", "follow-62", "66")) + assert result == {"comment": "Q1와 Q1", "improvements": ["문항 3 보완", None]} diff --git a/tests/core/common/feedback/test_strict_tool_http.py b/tests/core/common/feedback/test_strict_tool_http.py new file mode 100644 index 0000000..8cdf443 --- /dev/null +++ b/tests/core/common/feedback/test_strict_tool_http.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import json +from copy import deepcopy +from typing import Any + +import anthropic +import httpx +import pytest + +from app.core.common.company.fit.tools import SUBMIT_COMPANY_FIT_TOOL +from app.core.common.feedback.multi.tools import feedback_tool_for_job as multi_tool +from app.core.common.feedback.solo.tools import feedback_tool_for_job as solo_tool +from app.core.common.job_spec import JOBS +from app.outbound.adapters.anthropic_text_client_impl import AnthropicTextClientImpl + +_UNSUPPORTED = { + "minLength", + "maxLength", + "maxItems", + "uniqueItems", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", +} +_FEEDBACK_TOOLS = [ + pytest.param(factory(role), id=f"{mode}-{role}") + for mode, factory in (("solo", solo_tool), ("multi", multi_tool)) + for role in (None, *JOBS) +] + + +def _schema_errors(schema: dict[str, Any], path: str = "$") -> list[str]: + # Anthropic strict supports minItems 0/1 but no other length/count/numeric bounds. + errors = [f"{path}.{key}" for key in sorted(_UNSUPPORTED & schema.keys())] + if "minItems" in schema and schema["minItems"] not in (0, 1): + errors.append(f"{path}.minItems") + if schema.get("type") == "object" and schema.get("additionalProperties") is not False: + errors.append(f"{path}.additionalProperties") + for name, child in schema.get("properties", {}).items(): + errors.extend(_schema_errors(child, f"{path}.{name}")) + for name, child in schema.get("$defs", {}).items(): + errors.extend(_schema_errors(child, f"{path}.$defs.{name}")) + if isinstance(schema.get("items"), dict): + errors.extend(_schema_errors(schema["items"], f"{path}.items")) + for key in ("anyOf", "allOf"): + for index, child in enumerate(schema.get(key, [])): + errors.extend(_schema_errors(child, f"{path}.{key}[{index}]")) + return errors + + +@pytest.mark.parametrize("tool", [pytest.param(SUBMIT_COMPANY_FIT_TOOL, id="company"), *_FEEDBACK_TOOLS]) +async def test_real_sdk_sends_supported_strict_tools_without_network(tool: dict[str, Any]) -> None: + sent: list[dict[str, Any]] = [] + original = deepcopy(tool) + + def handle(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + sent.append(body) + errors = _schema_errors(body["tools"][0]["input_schema"]) + if errors: + return httpx.Response( + 400, + json={ + "type": "error", + "error": {"type": "invalid_request_error", "message": f"Unsupported strict schema: {errors}"}, + }, + ) + return httpx.Response( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "stub", + "content": [{"type": "tool_use", "id": "tool_test", "name": tool["name"], "input": {}}], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + + adapter = AnthropicTextClientImpl("test") + await adapter._client.close() + adapter._client = anthropic.AsyncAnthropic( + api_key="test", + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handle)), + ) + try: + result = await adapter.call( + model="stub", + system="test", + messages=[{"role": "user", "content": "자료"}], + tools=[tool], + tool_choice={"type": "tool", "name": tool["name"]}, + ) + finally: + await adapter.close() + assert result.stop_reason == "tool_use" + assert len(sent) == 1 + assert sent[0]["tools"] == [original] + assert sent[0]["tools"][0]["strict"] is True + assert tool == original + + +def test_schema_guard_checks_nested_constraints_and_supported_min_items() -> None: + schema = { + "type": "object", + "additionalProperties": False, + "properties": { + "text": {"anyOf": [{"type": "string", "maxLength": 200}, {"type": "null"}]}, + "items": {"type": "array", "maxItems": 3, "items": {"type": "string"}}, + "ids": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + }, + } + assert _schema_errors(schema) == ["$.text.anyOf[0].maxLength", "$.items.maxItems"] diff --git a/tests/regression/dev_roles/golden/call__feedback_multi.json b/tests/regression/dev_roles/golden/call__feedback_multi.json index c977676..1e0df3a 100644 --- a/tests/regression/dev_roles/golden/call__feedback_multi.json +++ b/tests/regression/dev_roles/golden/call__feedback_multi.json @@ -46,6 +46,10 @@ ], "type": "object" }, + "follow_up_question": { + "description": "현재 질문과 실제 답변의 근거·예외·트레이드오프를 더 확인하는 예상 꼬리질문 한 문장. 1~200자 한국어로 쓴다.", + "type": "string" + }, "improvements": { "description": "개선점. 질문 의도 중 답변이 다루지 않은 부분은 반드시 여기에 포함한다.", "items": { @@ -128,7 +132,8 @@ "improvements", "comment", "depth_evidence", - "scores" + "scores", + "follow_up_question" ], "type": "object" }, diff --git a/tests/regression/dev_roles/golden/call__feedback_solo.json b/tests/regression/dev_roles/golden/call__feedback_solo.json index 0ee4b61..dfbb051 100644 --- a/tests/regression/dev_roles/golden/call__feedback_solo.json +++ b/tests/regression/dev_roles/golden/call__feedback_solo.json @@ -46,6 +46,10 @@ ], "type": "object" }, + "follow_up_question": { + "description": "현재 질문과 실제 답변의 근거·예외·트레이드오프를 더 확인하는 예상 꼬리질문 한 문장. 1~200자 한국어로 쓴다.", + "type": "string" + }, "improvements": { "description": "개선점. 질문 의도 중 답변이 다루지 않은 부분은 반드시 여기에 포함한다.", "items": { @@ -128,7 +132,8 @@ "improvements", "comment", "depth_evidence", - "scores" + "scores", + "follow_up_question" ], "type": "object" }, diff --git a/tests/regression/dev_roles/golden/feedback_multi.txt b/tests/regression/dev_roles/golden/feedback_multi.txt index 6756fe6..39fc694 100644 --- a/tests/regression/dev_roles/golden/feedback_multi.txt +++ b/tests/regression/dev_roles/golden/feedback_multi.txt @@ -179,6 +179,10 @@ depth와 specificity는 질문한 면접관의 직책 관점으로 해석한다. - comment는 해당 면접관 시점의 한 문장짜리 총평이다. [문항별 피드백 feedbacks] +- follow_up_question: 실제 답변의 근거·예외·트레이드오프 중 하나를 더 확인하는 예상 꼬리질문 한 문장. + 현재 질문을 단순 반복하지 말고, 답변에 없는 경험·수치·구현을 했다고 전제하지 마라. + 질문만 200자 이내 한국어로 쓰고 점수·모범답안·목록 번호는 쓰지 마라. + 이 질문은 복습용이며 실제 면접에 삽입하거나 이미 출제된 질문이라고 주장하지 마라. - model_answer: 질문 의도를 충족하는 40~100자의 짧은 예시 답안. 사용자에게 보여주는 예시이지 채점 기준이 아니다. 해당 면접관의 직책 관점에서 쓰고, 제공되지 않은 사실을 지어내지 마라. diff --git a/tests/regression/dev_roles/golden/feedback_solo.txt b/tests/regression/dev_roles/golden/feedback_solo.txt index cca6a27..3d48d4f 100644 --- a/tests/regression/dev_roles/golden/feedback_solo.txt +++ b/tests/regression/dev_roles/golden/feedback_solo.txt @@ -147,6 +147,10 @@ consistency — 일관성: 면접 전체에서 한 번만 매긴다(overall.cons 근거가 부족한 것은 모순이 아니다. 근거 부족은 specificity에서 판단한다. [문항별 피드백] +- follow_up_question: 실제 답변의 근거·예외·트레이드오프 중 하나를 더 확인하는 예상 꼬리질문 한 문장. + 현재 질문을 단순 반복하지 말고, 답변에 없는 경험·수치·구현을 했다고 전제하지 마라. + 질문만 200자 이내 한국어로 쓰고 점수·모범답안·목록 번호는 쓰지 마라. + 이 질문은 복습용이며 실제 면접에 삽입하거나 이미 출제된 질문이라고 주장하지 마라. - model_answer: 질문 의도를 충족하는 40~100자의 짧은 예시 답안. 사용자에게 '이렇게 답할 수 있다'고 보여주는 예시이지 채점 기준이 아니다. 제공되지 않은 프로젝트 사실이나 구현을 지어내지 마라.