diff --git a/docs/IDEAHUB_INTEGRATION.md b/docs/IDEAHUB_INTEGRATION.md index 47e70aa..cdad812 100644 --- a/docs/IDEAHUB_INTEGRATION.md +++ b/docs/IDEAHUB_INTEGRATION.md @@ -56,6 +56,88 @@ both cases, review the YAML before spending substantial compute. ## Useful flags + constraints: + compute: cpu_only # For AI research + budget: 150 # Typical API costs, USD (numeric per schema) + + expected_outputs: [...] + evaluation_criteria: [...] +``` + +**GPT-4's Role:** +- **Domain Classification**: Infers appropriate domain from tags and content +- **Hypothesis Extraction**: Formulates testable hypothesis from description +- **Methodology Design**: Proposes experimental steps, baselines, and metrics +- **Constraint Estimation**: Sets realistic compute, time, and budget constraints +- **Output Specification**: Defines expected results and evaluation criteria + +### 3. Validation & Saving + +The converted YAML is: +1. Validated against the schema +2. Enhanced with metadata (source, source_url) +3. Saved with a sanitized filename derived from the title + +## Examples + +### Example 1: AI/LLM Research + +**IdeaHub URL:** https://hypogenic.ai/ideahub/idea/HGVv4Z0ALWVHZ9YsstWT + +**IdeaHub Content:** +- Title: "Do LLMs differentiate epistemic belief from non-epistemic belief?" +- Description: Research on whether LLMs exhibit distinct types of beliefs +- Tags: Psychology, LLM behavior + +**Converted YAML:** +```yaml +idea: + title: "Evaluating Epistemic vs Non-Epistemic Belief Differentiation in LLMs" + domain: artificial_intelligence + + hypothesis: | + LLMs demonstrate measurable differences in representing epistemic beliefs + (knowledge-based) versus non-epistemic beliefs (religious, moral), + similar to human cognitive patterns. + + methodology: + approach: "Comparative prompt-based evaluation" + steps: + - "Design prompts testing epistemic beliefs (factual knowledge)" + - "Design prompts testing non-epistemic beliefs (values, preferences)" + - "Run across multiple LLMs (GPT-4, Claude, Gemini)" + - "Analyze response patterns and confidence levels" + - "Compare with human baseline from Vesga et al. (2025)" + + baselines: + - "Human belief differentiation patterns from psychology research" + - "Zero-shot vs few-shot prompting" + + metrics: + - "Belief type classification accuracy" + - "Confidence level differences" + - "Response consistency across similar prompts" +``` + +### Example 2: Complete Workflow + +```bash +# 1. Fetch idea from IdeaHub +python src/cli/fetch_from_ideahub.py \ + https://hypogenic.ai/ideahub/idea/ABC123 \ + --submit + +# Output: idea_id_20250103_120000_abc123de + +# 2. (Optional) Add resources to workspace +cd workspace/idea-id-20250103-120000-abc123de +# Add datasets, papers, etc. +git add . && git commit -m "Add resources" && git push +cd ../.. + +# 3. Run the research +python src/core/runner.py idea_id_20250103_120000_abc123de +``` | Flag | Purpose | | --- | --- | | `--output PATH` | Write the converted YAML to a specific path | diff --git a/ideas/examples/ai_agent_tool_use.yaml b/ideas/examples/ai_agent_tool_use.yaml index 8c518a8..847dd61 100644 --- a/ideas/examples/ai_agent_tool_use.yaml +++ b/ideas/examples/ai_agent_tool_use.yaml @@ -68,7 +68,7 @@ idea: compute: cpu_only time_limit: 10800 # 3 hours memory: "16GB" - budget: "$150" + budget: 150 # USD; numeric per schema (formatted as $%.2f downstream) dependencies: - "openai>=1.0.0" - "anthropic>=0.18.0" diff --git a/ideas/examples/ai_chain_of_thought_evaluation.yaml b/ideas/examples/ai_chain_of_thought_evaluation.yaml index 4cc9548..4e4b287 100644 --- a/ideas/examples/ai_chain_of_thought_evaluation.yaml +++ b/ideas/examples/ai_chain_of_thought_evaluation.yaml @@ -74,7 +74,7 @@ idea: compute: cpu_only # Using APIs, no GPU needed time_limit: 7200 # 2 hours (rate limiting consideration) memory: "8GB" - budget: "$50" # Estimated API costs + budget: 50 # Estimated API costs in USD (numeric per schema) dependencies: - "openai>=1.0.0" - "anthropic>=0.18.0" diff --git a/ideas/schema.yaml b/ideas/schema.yaml index 174497b..72e120b 100644 --- a/ideas/schema.yaml +++ b/ideas/schema.yaml @@ -432,6 +432,15 @@ properties: type: string description: "Who submitted this idea" + source: + type: string + description: "Where the idea came from (auto-generated)" + examples: ["IdeaHub"] + + source_url: + type: string + description: "Original URL the idea was fetched from (auto-generated)" + created_at: type: string format: date-time diff --git a/src/cli/fetch_from_ideahub.py b/src/cli/fetch_from_ideahub.py index 2e41de2..5a11fdd 100644 --- a/src/cli/fetch_from_ideahub.py +++ b/src/cli/fetch_from_ideahub.py @@ -10,6 +10,9 @@ import os import re import json +import shlex +import shutil +import subprocess from pathlib import Path import requests from bs4 import BeautifulSoup @@ -182,18 +185,28 @@ def _convert_without_llm(ideahub_content: dict) -> dict: Returns: Dictionary with 'parsed' and 'yaml_string' keys """ - title = ideahub_content.get('title') or 'Untitled IdeaHub Idea' + title = (ideahub_content.get('title') or '').strip() or 'Untitled IdeaHub Idea' description = ideahub_content.get('description', '') tags = ideahub_content.get('tags', []) url = ideahub_content.get('url', '') + # The schema bounds title at 10..200 characters, and this is the last + # resort -- there is no further path to fall through to -- so a scraped + #

that is too short or too long has to be made to fit here rather + # than failing validation after the fact. + if len(title) < 10: + title = f"IdeaHub idea: {title}" + if len(title) > 200: + title = title[:197] + '...' + # Infer domain from content domain = _infer_domain(title, description, tags) - # Use description as hypothesis, ensuring minimum 20 chars + # Use description as hypothesis, ensuring the schema's 20-char minimum. + # The prefix plus a >=10-char title always clears it. hypothesis = description.strip() if len(hypothesis) < 20: - hypothesis = f"Investigate: {title}" + hypothesis = f"Investigate the research question: {title}" # Truncate very long hypotheses to keep it reasonable if len(hypothesis) > 500: hypothesis = hypothesis[:497] + '...' @@ -230,39 +243,281 @@ def _convert_without_llm(ideahub_content: dict) -> dict: return {'parsed': idea_data, 'yaml_string': yaml_string} -def convert_to_yaml(ideahub_content: dict) -> dict: +# CLI commands per provider (mirrors agents/manifest_trimmer.py) +CLI_COMMANDS = { + "claude": "claude -p", + "codex": "codex exec", + "gemini": "gemini", +} + +CONVERSION_SYSTEM_PROMPT = ( + "You are a research assistant that formats research ideas into minimal YAML. " + "Only include information explicitly provided - do not invent datasets, methods, " + "or metrics. Return valid YAML without markdown formatting." +) + + +def _extract_yaml(text: str) -> str: + """Pull the YAML document out of a raw LLM or CLI response.""" + fence = re.search(r"```ya?ml\s*\n(.*?)```", text, re.DOTALL) + candidate = fence.group(1) if fence else text.replace("```", "") + + # Drop any preamble the CLI printed before the document itself + match = re.search(r"^idea:", candidate, re.MULTILINE) + if match: + candidate = candidate[match.start():] + + return candidate.strip() + + +REQUIRED_IDEA_FIELDS = ('title', 'domain', 'hypothesis') + +# Fields the conversion prompt permits the model to emit. Deliberately much +# narrower than ideas/schema.yaml: IdeaHub pages carry community-submitted, +# untrusted content, so anything outside this set is dropped rather than +# written to disk. +# +# The exclusions that matter are the schema blocks that are *contractual* +# rather than advisory, and so would act on the host if a page's text talked +# the model into emitting them: +# local_resources -- staged into the workspace, and its host paths are +# written to ideas/mounts/.txt for docker/run.sh to +# mount; functions marked required_for_evaluation are +# imported and called +# evaluation -- transcribed verbatim into scoring/targets.json +# comments -- drives --comment-mode edits against an existing workspace +# methodology/expected_outputs/evaluation_criteria are excluded too: the +# prompt already tells the model not to produce them, so their presence means +# the model departed from its instructions. +CONVERTER_ALLOWED_IDEA_FIELDS = frozenset({ + 'title', 'domain', 'hypothesis', 'background', 'constraints', 'metadata', +}) + + +def _strip_disallowed_fields(parsed: dict) -> list: """ - Use GPT to convert IdeaHub content to NeuriCo YAML format. + Drop any idea field the converter is not allowed to emit, in place. - Args: - ideahub_content: Dictionary with IdeaHub content + Stripping rather than rejecting: a disallowed field means the response is + untrustworthy in that specific spot, not that the whole conversion is + worthless, and falling through would land on the far lossier template + path. Removing the field makes it unreachable while keeping the good + conversion. Callers re-render the YAML from this dict, so a stripped field + never reaches the file. Returns: - Dictionary in NeuriCo format + Sorted list of removed field names (empty when nothing was dropped). """ - print("\n🤖 Converting to NeuriCo format using GPT...") + idea = parsed['idea'] + removed = sorted(set(idea) - CONVERTER_ALLOWED_IDEA_FIELDS) + for field in removed: + del idea[field] + return removed - # Check for an API key: prefer OpenRouter (the repo default), fall back - # to a direct OpenAI key. - openrouter_key = os.getenv('OPENROUTER_KEY') or os.getenv('OPENROUTER_API_KEY') - api_key = openrouter_key or os.getenv('OPENAI_API_KEY') - if not api_key: - print("ℹī¸ No OPENROUTER_KEY or OPENAI_API_KEY set — using template-based conversion instead.") - return _convert_without_llm(ideahub_content) - try: - from openai import OpenAI - except ImportError: - print("ℹī¸ openai package not installed — using template-based conversion instead.") - return _convert_without_llm(ideahub_content) +def _is_structurally_complete(parsed) -> bool: + """ + Cheap structural gate used to decide whether a candidate parse is a + NeuriCo idea rather than merely well-formed YAML. + + Requires 'idea' to be the *only* top-level key, so a document followed by + provider commentary that happens to parse as a mapping ("Total tokens: + 812") is rejected and the trailing line gets trimmed instead of retained. + Then requires every schema-required field to be present and non-empty, so + a partial response -- a title-only idea, or one truncated by max_tokens -- + is rejected and the caller can fall through to the next conversion path. + """ + if not isinstance(parsed, dict) or set(parsed) != {'idea'}: + return False - if openrouter_key: - client = OpenAI(api_key=api_key, base_url="https://openrouter.ai/api/v1") - model_name = "openai/gpt-4.1" - else: - client = OpenAI(api_key=api_key) - model_name = "gpt-4.1" + idea = parsed['idea'] + if not isinstance(idea, dict): + return False + return all( + isinstance(idea.get(field), str) and idea[field].strip() + for field in REQUIRED_IDEA_FIELDS + ) + + +def _parse_idea_yaml(yaml_content: str) -> tuple: + """ + Parse an idea YAML document, tolerating trailing chatter that CLI agents + append after the document (token counts, closing remarks). Trims one line + at a time from the end until the text parses as a structurally complete + NeuriCo idea, then validates it against the full schema. + + Conversion validates the resulting idea, not just YAML syntax: an + incomplete response raises, and convert_to_yaml() falls through to the + next path rather than saving a half-formed idea. + + Returns: + (parsed_dict, cleaned_yaml_string) + + Raises: + ValueError: If no complete, valid idea document can be recovered. + """ + from core.idea_manager import validate_idea_spec + + lines = yaml_content.split("\n") + best_candidate = None + while lines: + text = "\n".join(lines).strip() + if text: + try: + parsed = yaml.safe_load(text) + except yaml.YAMLError: + parsed = None + if best_candidate is None and isinstance(parsed, dict) and 'idea' in parsed: + # Longest text that looked like an idea document; kept only to + # explain the failure if nothing turns out to be complete. + best_candidate = parsed + if _is_structurally_complete(parsed): + # Drop fields outside the converter's contract before + # validating, so a stripped field can never be what makes the + # idea valid. + removed = _strip_disallowed_fields(parsed) + for field in removed: + print(f" ⚠ī¸ Dropped disallowed field 'idea.{field}' from " + f"the converted idea (not permitted from fetched " + f"content)") + if removed: + # Keep the returned string consistent with the dict; every + # caller re-renders today, but a stale pair is a trap. + text = _dump_idea_yaml(parsed) + # Full schema validation runs once, on the winning candidate, + # so failures name the offending field instead of degrading + # into "no document found". + report = validate_idea_spec(parsed) + if not report['valid']: + raise ValueError( + "converted idea failed validation: " + + "; ".join(report['errors']) + ) + # Warnings are deliberately not printed here -- main() prints + # them once, against the finalized dict that gets written. + return parsed, text + lines.pop() + + if best_candidate is not None: + idea = best_candidate.get('idea') + if not isinstance(idea, dict): + detail = "'idea' is not a mapping" + else: + missing = [ + field for field in REQUIRED_IDEA_FIELDS + if not (isinstance(idea.get(field), str) and idea[field].strip()) + ] + detail = f"missing or empty required field(s): {', '.join(missing)}" + raise ValueError( + f"response contained an 'idea:' document but it was incomplete " + f"({detail})" + ) + raise ValueError("no valid 'idea:' YAML document found in the response") + + +def _dump_idea_yaml(idea_data: dict) -> str: + """ + Dump idea data back to YAML, keeping multi-line text as literal blocks so the + output stays as readable as what the LLM produced. + """ + class _IdeaDumper(yaml.SafeDumper): + pass + + def _represent_str(dumper, value): + style = '|' if '\n' in value else None + return dumper.represent_scalar('tag:yaml.org,2002:str', value, style=style) + + _IdeaDumper.add_representer(str, _represent_str) + + return yaml.dump(idea_data, Dumper=_IdeaDumper, default_flow_style=False, + sort_keys=False, allow_unicode=True) + + +def _apply_source_metadata(idea_data: dict, ideahub_content: dict) -> dict: + """ + Apply provenance fields the converter -- not the model -- is authoritative + for. + + source and source_url are overwritten, never merged: the converter knows + it fetched this idea from IdeaHub and knows the exact URL it fetched, so a + model-emitted value is at best redundant and at worst a hallucinated URL + saved as if it were the real origin. A model value that disagrees is + reported before being replaced. + + author is different: it is content, extracted from the page by the model, + and the scraped author is only a fallback for the slot the model left + empty -- so it keeps setdefault semantics. + + Applied to the parsed dict before the YAML string is regenerated, so the + written file and the dict handed to submit_idea() always agree. + """ + if 'idea' not in idea_data: + idea_data = {'idea': idea_data} + + metadata = idea_data['idea'].setdefault('metadata', {}) + + authoritative = { + 'source': 'IdeaHub', + 'source_url': ideahub_content.get('url', ''), + } + for field, value in authoritative.items(): + existing = metadata.get(field) + if existing is not None and existing != value: + print(f" ⚠ī¸ Ignoring model-supplied metadata.{field} " + f"({existing!r}); using {value!r}") + metadata[field] = value + + author = ideahub_content.get('author') + if author: + metadata.setdefault('author', author) + + return idea_data + + +def _convert_with_cli(prompt: str, provider: str, timeout: int = 300) -> dict: + """ + Convert via a local agent CLI (codex/claude/gemini). + + These authenticate with their own login (e.g. your ChatGPT account for + codex), so this path still works when OPENAI_API_KEY is unset or has + exhausted its quota. + """ + cmd = CLI_COMMANDS[provider] + binary = shlex.split(cmd)[0] + if shutil.which(binary) is None: + raise RuntimeError(f"'{binary}' not found on PATH") + + print(f" Calling {provider} CLI ({cmd})...") + + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + if provider == "gemini": + env["GEMINI_CLI_IDE_DISABLE"] = "1" + + result = subprocess.run( + shlex.split(cmd), + input=f"{CONVERSION_SYSTEM_PROMPT}\n\n{prompt}", + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + env=env, + timeout=timeout, + ) + + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip()[-500:] + raise RuntimeError(f"exited with code {result.returncode}: {detail}") + + parsed, yaml_content = _parse_idea_yaml(_extract_yaml(result.stdout)) + print(" ✓ Conversion complete") + + return {'parsed': parsed, 'yaml_string': yaml_content} + + +def _build_conversion_prompt(ideahub_content: dict) -> str: + """Build the IdeaHub -> NeuriCo YAML conversion prompt.""" # Read schema for reference schema_path = Path(__file__).parent.parent.parent / "ideas" / "schema.yaml" with open(schema_path, 'r', encoding='utf-8') as f: @@ -361,81 +616,141 @@ def convert_to_yaml(ideahub_content: dict) -> dict: ``` """ - try: - print(" Calling GPT API...") - response = client.chat.completions.create( - model=model_name, - messages=[ - { - "role": "system", - "content": "You are a research assistant that formats research ideas into minimal YAML. Only include information explicitly provided - do not invent datasets, methods, or metrics. Return valid YAML without markdown formatting." - }, - { - "role": "user", - "content": prompt - } - ], - temperature=0.1, # Lower temperature for more conservative output - max_tokens=2000 # Reduced since we want minimal output - ) + return prompt + + +def _resolve_api_key() -> tuple: + """ + Resolve the LLM API key, preferring OpenRouter (the repo default) over a + direct OpenAI key. + + Returns: + (api_key, use_openrouter) -- api_key is None when neither is set. + """ + openrouter_key = os.getenv('OPENROUTER_KEY') or os.getenv('OPENROUTER_API_KEY') + if openrouter_key: + return openrouter_key, True + return os.getenv('OPENAI_API_KEY'), False + + +def _convert_with_openai(prompt: str, api_key: str, use_openrouter: bool = False) -> dict: + """Convert via the OpenAI API. Raises on auth, quota, or parse failure.""" + from openai import OpenAI + + if use_openrouter: + client = OpenAI(api_key=api_key, base_url="https://openrouter.ai/api/v1") + model_name = "openai/gpt-4.1" + else: + client = OpenAI(api_key=api_key) + model_name = "gpt-4.1" - yaml_content = response.choices[0].message.content.strip() + print(" Calling GPT API...") + response = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": CONVERSION_SYSTEM_PROMPT}, + {"role": "user", "content": prompt} + ], + temperature=0.1, # Lower temperature for more conservative output + max_tokens=2000 # Reduced since we want minimal output + ) + + raw = response.choices[0].message.content.strip() + parsed, yaml_content = _parse_idea_yaml(_extract_yaml(raw)) + print(" ✓ Conversion complete") - # Remove markdown code fences if present - yaml_content = re.sub(r'^```ya?ml\s*\n', '', yaml_content) - yaml_content = re.sub(r'\n```\s*$', '', yaml_content) - yaml_content = yaml_content.strip() + return {'parsed': parsed, 'yaml_string': yaml_content} - print(" ✓ Conversion complete") - # Parse YAML to validate +def convert_to_yaml(ideahub_content: dict, provider: str = None) -> dict: + """ + Convert IdeaHub content to NeuriCo YAML format. + + Tries three paths in order, falling through on failure: + 1. The OpenAI-compatible API, if OPENROUTER_KEY (preferred, the repo + default) or OPENAI_API_KEY is set. + 2. The local agent CLI for `provider` (default codex), which uses its own + login rather than an API key -- so it survives a quota error. + 3. A template-based conversion, which preserves far less of the source + idea (no citations, keyword-guessed domain). + + Args: + ideahub_content: Dictionary with IdeaHub content + provider: Agent CLI to fall back to (claude, codex, gemini) + + Returns: + Dictionary with 'parsed' and 'yaml_string' keys + """ + prompt = _build_conversion_prompt(ideahub_content) + cli_provider = provider or "codex" + + def _finalize(result: dict) -> dict: + """ + Drop placeholder authors, attach provenance metadata, and re-render the + YAML from the same dict. + + The placeholder drop runs before _apply_source_metadata so a scraped + author can still fill the slot the model left as 'Unknown'. + """ + idea_data = result['parsed'] + if 'idea' not in idea_data: + idea_data = {'idea': idea_data} + idea_data = _drop_placeholder_author(idea_data) + idea_data = _apply_source_metadata(idea_data, ideahub_content) + return {'parsed': idea_data, 'yaml_string': _dump_idea_yaml(idea_data)} + + api_key, use_openrouter = _resolve_api_key() + if api_key: + print("\n🤖 Converting to NeuriCo format using GPT...") try: - parsed = yaml.safe_load(yaml_content) - except yaml.YAMLError as e: - print(f"⚠ī¸ Warning: Generated YAML may have issues: {e}") - print(" Attempting to fix...") - # Try to parse anyway - parsed = yaml.safe_load(yaml_content) + return _finalize(_convert_with_openai(prompt, api_key, use_openrouter)) + except ImportError: + print("⚠ī¸ openai package not installed.") + except Exception as e: + print(f"⚠ī¸ GPT API call failed: {e}") + else: + print("\nℹī¸ No OPENROUTER_KEY or OPENAI_API_KEY set.") - parsed, yaml_content = _drop_placeholder_author(parsed, yaml_content) - # Return both parsed data and the raw YAML string - return {'parsed': parsed, 'yaml_string': yaml_content} + if cli_provider in CLI_COMMANDS: + print(f"🤖 Converting to NeuriCo format using the {cli_provider} CLI...") + try: + return _finalize(_convert_with_cli(prompt, cli_provider)) + except Exception as e: + print(f"⚠ī¸ {cli_provider} CLI conversion failed: {e}") - except Exception as e: - print(f"⚠ī¸ GPT API call failed: {e}") - print(" Falling back to template-based conversion.") - return _convert_without_llm(ideahub_content) + print(" Falling back to template-based conversion.") + return _finalize(_convert_without_llm(ideahub_content)) -def _drop_placeholder_author(parsed: dict, yaml_string: str) -> tuple: +def _drop_placeholder_author(idea_data: dict) -> dict: """ Remove metadata.author when the model emitted the 'Unknown' placeholder - despite being told to omit it. Regenerates the YAML string only when a - drop actually happened, so faithful conversions stay byte-identical. + despite being told to omit it. The caller re-renders the YAML from this + dict, so only the parsed structure is touched here. """ try: - metadata = parsed['idea']['metadata'] + metadata = idea_data['idea']['metadata'] author = metadata.get('author') except (KeyError, TypeError): - return parsed, yaml_string + return idea_data if isinstance(author, str) and author.strip().lower() in ('unknown', ''): del metadata['author'] if not metadata: - del parsed['idea']['metadata'] - yaml_string = yaml.dump(parsed, default_flow_style=False, - sort_keys=False, allow_unicode=True) - return parsed, yaml_string + del idea_data['idea']['metadata'] + return idea_data -def save_yaml_file(result: dict, url: str, author: str = None) -> Path: +def save_yaml_file(result: dict, url: str) -> Path: """ Save the idea as a YAML file. + Provenance metadata (source, source_url, author) is already applied by + convert_to_yaml(), so 'yaml_string' here is a faithful rendering of 'parsed'. + Args: result: Dictionary with 'parsed' and 'yaml_string' keys - url: Original IdeaHub URL - author: Optional author name from IdeaHub + url: Original IdeaHub URL (used for the filename fallback) Returns: Path to saved file @@ -458,22 +773,6 @@ def save_yaml_file(result: dict, url: str, author: str = None) -> Path: else: filename = "ideahub_idea" - # Add metadata about source to the parsed data (for submission later) - if 'idea' not in idea_data: - idea_data = {'idea': idea_data} - - if 'metadata' not in idea_data['idea']: - idea_data['idea']['metadata'] = {} - - idea_data['idea']['metadata']['source'] = 'IdeaHub' - idea_data['idea']['metadata']['source_url'] = url - - if author and 'author' not in idea_data['idea']['metadata']: - idea_data['idea']['metadata']['author'] = author - - # Update the result - result['parsed'] = idea_data - # Save to ideas/ directory ideas_dir = Path(__file__).parent.parent.parent / "ideas" ideas_dir.mkdir(exist_ok=True) @@ -534,7 +833,7 @@ def main(): "--provider", choices=["claude", "gemini", "codex"], default=None, - help="AI provider for repo naming and --run execution" + help="AI provider for YAML conversion fallback, repo naming, and --run execution (default: codex for conversion)" ) parser.add_argument( "--no-hash", @@ -598,8 +897,21 @@ def main(): if ideahub_content.get('title'): print(f"\n✓ Found idea: {ideahub_content['title']}") - # Step 2: Convert with GPT - result = convert_to_yaml(ideahub_content) + # Step 2: Convert (GPT, then the provider's CLI, then a template) + result = convert_to_yaml(ideahub_content, provider=args.provider) + + # Step 2b: Validate the finished idea before it is written. + # + # This runs whether or not --submit was passed. The LLM paths already + # validated at parse time, but the template fallback has no further path + # to fall through to, and provenance metadata is applied after parsing -- + # so the last word on validity belongs here, on exactly the dict that is + # about to be saved. + from core.idea_manager import validate_idea_spec + + report = validate_idea_spec(result['parsed']) + for warning in report['warnings']: + print(f"⚠ī¸ {warning}") # Step 3: Save file if args.output: @@ -609,7 +921,18 @@ def main(): with open(output_path, 'w', encoding='utf-8') as f: f.write(result['yaml_string']) else: - output_path = save_yaml_file(result, args.url, author=ideahub_content.get('author')) + output_path = save_yaml_file(result, args.url) + + if not report['valid']: + # Written anyway so the incomplete draft can be hand-edited, but never + # reported as a success -- and never submitted. + print(f"\n📝 Incomplete idea written to: {output_path}") + print("\n❌ Conversion did not produce a complete NeuriCo idea:") + for error in report['errors']: + print(f" - {error}") + print("\n Fix the file above, then submit it with:") + print(f" python src/cli/submit.py {output_path}") + sys.exit(1) print(f"\n✅ Idea saved to: {output_path}") diff --git a/src/core/idea_manager.py b/src/core/idea_manager.py index 3697124..9fe647d 100644 --- a/src/core/idea_manager.py +++ b/src/core/idea_manager.py @@ -14,6 +14,7 @@ import yaml import json import hashlib +import re import sys import os @@ -28,6 +29,336 @@ ) +def _check_mapping(value: Any, label: str, errors: List[str]) -> bool: + """Record an error unless value is a mapping. Returns True when it is. + + Callers gate further inspection on the return value: every consumer of + these blocks (prompt_generator, the agents) calls .get() on them, so a + non-mapping is a crash waiting to happen rather than a cosmetic problem. + """ + if not isinstance(value, dict): + errors.append(f"{label} must be a mapping, got {type(value).__name__}") + return False + return True + + +def _check_string(value: Any, label: str, errors: List[str], + min_length: int = None, max_length: int = None) -> bool: + """Record an error unless value is a string within the schema's bounds.""" + if not isinstance(value, str): + errors.append(f"{label} must be a string, got {type(value).__name__}") + return False + if min_length is not None and len(value) < min_length: + errors.append(f"{label} must be at least {min_length} characters " + f"(got {len(value)})") + return False + if max_length is not None and len(value) > max_length: + errors.append(f"{label} must be at most {max_length} characters " + f"(got {len(value)})") + return False + return True + + +def _check_list(value: Any, label: str, errors: List[str], + item_type: type = None, item_label: str = "item") -> bool: + """Record an error unless value is a list, optionally of item_type.""" + if not isinstance(value, list): + errors.append(f"{label} must be a list, got {type(value).__name__}") + return False + if item_type is not None: + for idx, item in enumerate(value): + if not isinstance(item, item_type): + errors.append( + f"{label}[{idx}]: {item_label} must be " + f"{item_type.__name__}, got {type(item).__name__}") + return False + return True + + +def _validate_background(background: Any, errors: List[str], + warnings: List[str]) -> None: + """Validate idea.background against the schema.""" + if not _check_mapping(background, "background", errors): + return + + if 'description' in background: + _check_string(background['description'], "background.description", errors) + + # papers: each entry needs a description plus either a url or a path + if 'papers' in background and background['papers'] is not None: + if _check_list(background['papers'], "background.papers", errors): + for idx, paper in enumerate(background['papers']): + label = f"background.papers[{idx}]" + if not _check_mapping(paper, label, errors): + continue + if 'url' not in paper and 'path' not in paper: + errors.append(f"{label}: must provide either 'url' or 'path'") + if 'description' not in paper: + errors.append(f"{label}: missing required field 'description'") + + if 'datasets' in background and background['datasets'] is not None: + if _check_list(background['datasets'], "background.datasets", errors): + for idx, dataset in enumerate(background['datasets']): + label = f"background.datasets[{idx}]" + if not _check_mapping(dataset, label, errors): + continue + for field in ('name', 'source'): + if field not in dataset: + errors.append(f"{label}: missing required field '{field}'") + + if 'code_references' in background and background['code_references'] is not None: + if _check_list(background['code_references'], "background.code_references", errors): + for idx, ref in enumerate(background['code_references']): + label = f"background.code_references[{idx}]" + if not _check_mapping(ref, label, errors): + continue + for field in ('repo', 'description'): + if field not in ref: + errors.append(f"{label}: missing required field '{field}'") + + +def _validate_methodology(methodology: Any, errors: List[str]) -> None: + """Validate idea.methodology against the schema.""" + if not _check_mapping(methodology, "methodology", errors): + return + + if 'approach' in methodology: + _check_string(methodology['approach'], "methodology.approach", errors) + + for field in ('steps', 'baselines', 'metrics'): + if field in methodology and methodology[field] is not None: + _check_list(methodology[field], f"methodology.{field}", errors, + item_type=str, item_label="entry") + + +def _validate_constraints(constraints: Any, errors: List[str], + warnings: List[str]) -> None: + """Validate idea.constraints against the schema.""" + if not _check_mapping(constraints, "constraints", errors): + return + + if 'compute' in constraints: + valid_compute = ['cpu_only', 'gpu_required', 'multi_gpu', 'tpu', 'any'] + if constraints['compute'] not in valid_compute: + errors.append(f"Invalid compute constraint: {constraints['compute']}") + + if 'time_limit' in constraints: + # Range stays advisory: an out-of-range limit is a judgement call, not + # a structural fault, and nothing downstream breaks on it. + if not isinstance(constraints['time_limit'], int) or \ + isinstance(constraints['time_limit'], bool): + errors.append("time_limit must be an integer (seconds)") + elif constraints['time_limit'] < 60: + warnings.append("time_limit is very short (< 60 seconds)") + elif constraints['time_limit'] > 86400: + warnings.append("time_limit is very long (> 24 hours)") + + if 'memory' in constraints: + if _check_string(constraints['memory'], "constraints.memory", errors): + if not re.fullmatch(r'[0-9]+(GB|MB)', constraints['memory']): + errors.append( + f"constraints.memory must look like '8GB' or '512MB', " + f"got {constraints['memory']!r}") + + if 'budget' in constraints: + budget = constraints['budget'] + if isinstance(budget, bool) or not isinstance(budget, (int, float)): + errors.append( + f"constraints.budget must be a number, got {type(budget).__name__}") + elif budget < 0: + errors.append("constraints.budget must not be negative") + + if 'dependencies' in constraints and constraints['dependencies'] is not None: + _check_list(constraints['dependencies'], "constraints.dependencies", + errors, item_type=str, item_label="dependency") + + +def _validate_metadata(metadata: Any, errors: List[str]) -> None: + """Validate idea.metadata against the schema.""" + if not _check_mapping(metadata, "metadata", errors): + return + + for field in ('author', 'source', 'source_url', 'estimated_duration'): + if field in metadata and metadata[field] is not None: + _check_string(metadata[field], f"metadata.{field}", errors) + + for field in ('tags', 'related_ideas'): + if field in metadata and metadata[field] is not None: + _check_list(metadata[field], f"metadata.{field}", errors, + item_type=str, item_label="entry") + + if 'priority' in metadata: + valid_priorities = ['low', 'medium', 'high', 'urgent'] + if metadata['priority'] not in valid_priorities: + errors.append( + f"Invalid metadata.priority: {metadata['priority']}. " + f"Must be one of: {', '.join(valid_priorities)}") + + +def _validate_expected_outputs(expected_outputs: Any, errors: List[str], + warnings: List[str]) -> None: + """Validate idea.expected_outputs against the schema.""" + if not _check_list(expected_outputs, "expected_outputs", errors): + return + + if not expected_outputs: + warnings.append("expected_outputs is empty - agent will determine appropriate outputs") + return + + # The schema lists an enum, but output types are open-ended in practice: + # the shipped math/Lean examples declare 'proof' and + # 'computational_verification', and domains keep inventing their own. So + # an unrecognized type warns rather than fails -- the same treatment + # unknown domains already get. Structure (mapping, type, format) stays a + # hard requirement, since that is what consumers actually index into. + known_types = ['metrics', 'visualization', 'model', 'dataset', 'report', + 'code', 'analysis'] + for idx, output in enumerate(expected_outputs): + if not _check_mapping(output, f"Output {idx}", errors): + continue + if 'type' not in output: + errors.append(f"Output {idx}: missing 'type' field") + elif output['type'] not in known_types: + warnings.append(f"Output {idx}: unrecognized type " + f"{output['type']!r} (known types: " + f"{', '.join(known_types)})") + if 'format' not in output: + errors.append(f"Output {idx}: missing 'format' field") + + +def validate_idea_spec(idea_spec: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate an idea specification against the NeuriCo schema. + + Module-level so callers that only need validation (e.g. the IdeaHub + converter, which validates before anything is written) can reach it + without constructing an IdeaManager -- whose __init__ creates the + submitted/in_progress/completed directories as a side effect. + + Args: + idea_spec: Idea specification dictionary + + Returns: + Dictionary with keys: + - 'valid': bool + - 'errors': List of error messages + - 'warnings': List of warning messages + """ + errors = [] + warnings = [] + + # Check top-level structure + if not isinstance(idea_spec, dict) or 'idea' not in idea_spec: + errors.append("Missing top-level 'idea' key") + return {'valid': False, 'errors': errors, 'warnings': warnings} + + idea = idea_spec['idea'] + + # Every check below indexes into `idea`; a non-mapping here would turn + # membership tests into substring tests (or raise), so stop now. + if not _check_mapping(idea, "idea", errors): + return {'valid': False, 'errors': errors, 'warnings': warnings} + + # Required fields (v1.1 - reduced from v1.0) + required_fields = ['title', 'domain', 'hypothesis'] + for field in required_fields: + if field not in idea or not idea[field]: + errors.append(f"Missing required field: {field}") + + # Required-field types and lengths, per ideas/schema.yaml. Only checked + # when present and non-empty; absence is already an error above. + if idea.get('title'): + _check_string(idea['title'], "title", errors, min_length=10, max_length=200) + if idea.get('hypothesis'): + _check_string(idea['hypothesis'], "hypothesis", errors, min_length=20) + + # Validate domain + domain_is_string = True + if idea.get('domain'): + domain_is_string = _check_string(idea['domain'], "domain", errors) + + config_loader = ConfigLoader() + valid_domains = config_loader.get_valid_domains() + allow_unknown = config_loader.should_allow_unknown_domains() + + if domain_is_string and 'domain' in idea and idea['domain'] not in valid_domains: + if allow_unknown: + default_domain = config_loader.get_default_domain() + warnings.append( + f"Unknown domain '{idea['domain']}' will be treated as '{default_domain}'. " + f"Valid domains: {', '.join(valid_domains)}" + ) + else: + errors.append( + f"Invalid domain: {idea['domain']}. " + f"Must be one of: {', '.join(valid_domains)}" + ) + + if 'max_directions' in idea: + max_directions = idea['max_directions'] + if not isinstance(max_directions, int) or isinstance(max_directions, bool): + errors.append("max_directions must be an integer") + elif not 1 <= max_directions <= 10: + errors.append("max_directions must be between 1 and 10") + + if 'comments' in idea and idea['comments'] is not None: + _check_string(idea['comments'], "comments", errors) + + # Optional structured blocks. Each is a mapping or list downstream + # (prompt_generator and the agents call .get()/iterate on them), so a + # wrong type here becomes a crash mid-run rather than a bad prompt. + # + # An explicit null is treated as absent rather than as a type error: + # `constraints:` with nothing under it is ordinary YAML for "no + # constraints", and every consumer guards with `if constraints:`, so None + # is skipped safely. A *string* like `constraints: none` is not -- that + # reaches constraints.get('compute') and raises. Hence the isinstance + # checks below rather than a truthiness test. + if 'background' in idea and idea['background'] is not None: + _validate_background(idea['background'], errors, warnings) + + if 'methodology' in idea and idea['methodology'] is not None: + _validate_methodology(idea['methodology'], errors) + + if 'constraints' in idea and idea['constraints'] is not None: + _validate_constraints(idea['constraints'], errors, warnings) + + if 'metadata' in idea and idea['metadata'] is not None: + _validate_metadata(idea['metadata'], errors) + + # Validate expected outputs (optional in v1.1) + if 'expected_outputs' in idea and idea['expected_outputs'] is not None: + _validate_expected_outputs(idea['expected_outputs'], errors, warnings) + else: + warnings.append("No expected_outputs specified - agent will determine appropriate outputs based on research type") + + # Validate evaluation criteria + if 'evaluation_criteria' in idea and idea['evaluation_criteria'] is not None: + if _check_list(idea['evaluation_criteria'], "evaluation_criteria", + errors, item_type=str, item_label="criterion"): + if len(idea['evaluation_criteria']) == 0: + warnings.append("No evaluation criteria specified") + + # Validate local resources (contractual: path + usage required, + # missing paths are warnings until staging) + lr_errors, lr_warnings = validate_local_resources(idea) + errors.extend(lr_errors) + warnings.extend(lr_warnings) + + # Validate structured evaluation spec + ev_errors, ev_warnings = validate_evaluation_spec(idea) + errors.extend(ev_errors) + warnings.extend(ev_warnings) + + valid = len(errors) == 0 + + return { + 'valid': valid, + 'errors': errors, + 'warnings': warnings + } + + def resolve_ideas_dir(project_root: Optional[Path] = None) -> Path: """Resolve the ideas directory, honoring the NEURICO_IDEAS override. @@ -140,6 +471,9 @@ def validate_idea(self, idea_spec: Dict[str, Any]) -> Dict[str, Any]: """ Validate idea specification. + Delegates to the module-level validate_idea_spec() so the converter + and the submit path enforce exactly the same rules. + Args: idea_spec: Idea specification dictionary @@ -149,109 +483,7 @@ def validate_idea(self, idea_spec: Dict[str, Any]) -> Dict[str, Any]: - 'errors': List of error messages - 'warnings': List of warning messages """ - errors = [] - warnings = [] - - # Check top-level structure - if 'idea' not in idea_spec: - errors.append("Missing top-level 'idea' key") - return {'valid': False, 'errors': errors, 'warnings': warnings} - - idea = idea_spec['idea'] - - # Required fields (v1.1 - reduced from v1.0) - required_fields = ['title', 'domain', 'hypothesis'] - for field in required_fields: - if field not in idea or not idea[field]: - errors.append(f"Missing required field: {field}") - - # Validate domain - config_loader = ConfigLoader() - valid_domains = config_loader.get_valid_domains() - allow_unknown = config_loader.should_allow_unknown_domains() - - if 'domain' in idea and idea['domain'] not in valid_domains: - if allow_unknown: - default_domain = config_loader.get_default_domain() - warnings.append( - f"Unknown domain '{idea['domain']}' will be treated as '{default_domain}'. " - f"Valid domains: {', '.join(valid_domains)}" - ) - else: - errors.append( - f"Invalid domain: {idea['domain']}. " - f"Must be one of: {', '.join(valid_domains)}" - ) - - # Validate hypothesis length - if 'hypothesis' in idea and len(idea['hypothesis']) < 20: - warnings.append("Hypothesis is very short (< 20 characters). " - "Consider providing more detail.") - - if 'max_directions' in idea: - max_directions = idea['max_directions'] - if not isinstance(max_directions, int) or isinstance(max_directions, bool): - errors.append("max_directions must be an integer") - elif not 1 <= max_directions <= 10: - errors.append("max_directions must be between 1 and 10") - - # Validate expected outputs (optional in v1.1) - if 'expected_outputs' in idea: - if not isinstance(idea['expected_outputs'], list): - errors.append("expected_outputs must be a list") - elif len(idea['expected_outputs']) == 0: - warnings.append("expected_outputs is empty - agent will determine appropriate outputs") - else: - for idx, output in enumerate(idea['expected_outputs']): - if 'type' not in output: - errors.append(f"Output {idx}: missing 'type' field") - if 'format' not in output: - errors.append(f"Output {idx}: missing 'format' field") - else: - warnings.append("No expected_outputs specified - agent will determine appropriate outputs based on research type") - - # Validate constraints - if 'constraints' in idea: - constraints = idea['constraints'] - - if 'compute' in constraints: - valid_compute = ['cpu_only', 'gpu_required', 'multi_gpu', 'tpu', 'any'] - if constraints['compute'] not in valid_compute: - errors.append(f"Invalid compute constraint: {constraints['compute']}") - - if 'time_limit' in constraints: - if not isinstance(constraints['time_limit'], int): - errors.append("time_limit must be an integer (seconds)") - elif constraints['time_limit'] < 60: - warnings.append("time_limit is very short (< 60 seconds)") - elif constraints['time_limit'] > 86400: - warnings.append("time_limit is very long (> 24 hours)") - - # Validate evaluation criteria - if 'evaluation_criteria' in idea: - if not isinstance(idea['evaluation_criteria'], list): - errors.append("evaluation_criteria must be a list") - elif len(idea['evaluation_criteria']) == 0: - warnings.append("No evaluation criteria specified") - - # Validate local resources (contractual: path + usage required, - # missing paths are warnings until staging) - lr_errors, lr_warnings = validate_local_resources(idea) - errors.extend(lr_errors) - warnings.extend(lr_warnings) - - # Validate structured evaluation spec - ev_errors, ev_warnings = validate_evaluation_spec(idea) - errors.extend(ev_errors) - warnings.extend(ev_warnings) - - valid = len(errors) == 0 - - return { - 'valid': valid, - 'errors': errors, - 'warnings': warnings - } + return validate_idea_spec(idea_spec) def get_idea(self, idea_id: str) -> Optional[Dict[str, Any]]: """