Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 11 additions & 26 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches: [main, master]
paths:
- 'agent/**'
- 'cli/**'
- 'validator/**'
- 'spec/**'
- 'evals/**'
Expand All @@ -15,7 +15,7 @@ on:
pull_request:
branches: [main, master]
paths:
- 'agent/**'
- 'cli/**'
- 'validator/**'
- 'spec/**'
- 'evals/**'
Expand All @@ -37,21 +37,11 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4

- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'

- name: Install PyYAML
run: pip install pyyaml
- name: Install uv
uses: astral-sh/setup-uv@v5

- name: Regenerate all spec-derived files
run: |
# Generate system.md, README language spec, etc.
python spec/generate.py all

# Note: validator tokens are generated at compile-time via build.rs
# so we don't need tokens_generated.rs committed anymore
run: uv run spec/generate.py all

- name: Check for uncommitted changes
run: |
Expand Down Expand Up @@ -91,27 +81,22 @@ jobs:
restore-keys: |
${{ runner.os }}-cargo-

- name: Build validator
run: |
cd validator
cargo build --release
- name: Build validator library
run: cd validator && cargo build --release

- name: Build agent
- name: Build CLI
run: |
cd agent
cd cli
./setup.sh
cargo build --release

- name: Run validator tests
run: |
cd validator
cargo test
run: cd validator && cargo test

- name: Collect binaries
run: |
mkdir -p dist
cp validator/target/release/worldview-validate dist/
cp agent/target/release/worldview dist/
cp cli/target/release/worldview dist/
chmod +x dist/*

- name: Upload artifacts
Expand Down
30 changes: 10 additions & 20 deletions .github/workflows/evals.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,11 @@ jobs:
# For workflow_run, checkout the commit that triggered the build
ref: ${{ github.event.workflow_run.head_sha || github.sha }}

- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
cache-dependency-path: 'evals/requirements.txt'
- name: Install uv
uses: astral-sh/setup-uv@v5

- name: Install dependencies
run: |
pip install -r evals/requirements.txt
run: uv sync

- name: Download Worldview tools from build
if: ${{ github.event_name == 'workflow_run' }}
Expand Down Expand Up @@ -108,7 +103,7 @@ jobs:
mkdir -p results

# Build command
CMD="python -m evals run --verbose --output results"
CMD="uv run python -m evals run --verbose --output results"

# Add models
if [ "${{ steps.models.outputs.use_all }}" == "true" ]; then
Expand Down Expand Up @@ -147,7 +142,7 @@ jobs:
- name: Check for failures
run: |
# Extract success rate from JSON and fail if below threshold
python -c "
uv run python -c "
import json
with open('results/results.json') as f:
data = json.load(f)
Expand Down Expand Up @@ -182,16 +177,11 @@ jobs:
with:
ref: ${{ github.event.workflow_run.head_sha || github.sha }}

- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
cache-dependency-path: 'evals/requirements.txt'
- name: Install uv
uses: astral-sh/setup-uv@v5

- name: Install dependencies
run: |
pip install -r evals/requirements.txt
run: uv sync

- name: Download Worldview tools from build
if: ${{ github.event_name == 'workflow_run' }}
Expand Down Expand Up @@ -231,7 +221,7 @@ jobs:
mkdir -p results

# Build command
CMD="python -m evals write-eval --verbose --output results --agent-cli ./bin/worldview"
CMD="uv run python -m evals write-eval --verbose --output results --agent-cli ./bin/worldview"

# Add models (write eval only supports Claude models)
CMD="$CMD --all-models"
Expand Down Expand Up @@ -263,7 +253,7 @@ jobs:
- name: Check write eval results
if: steps.cli-check.outputs.available == 'true'
run: |
python -c "
uv run python -c "
import json
with open('results/write_results.json') as f:
data = json.load(f)
Expand Down
85 changes: 85 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
name: Release

on:
push:
tags:
- 'v*'

env:
CARGO_TERM_COLOR: always

jobs:
build:
strategy:
matrix:
include:
- target: aarch64-apple-darwin
os: macos-14
suffix: darwin-arm64
- target: x86_64-unknown-linux-gnu
os: ubuntu-latest
suffix: linux-x86_64
- target: aarch64-unknown-linux-gnu
os: ubuntu-24.04-arm
suffix: linux-arm64

runs-on: ${{ matrix.os }}

steps:
- uses: actions/checkout@v4

- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}

- name: Install uv
uses: astral-sh/setup-uv@v5

- name: Generate spec files
run: uv run spec/generate.py system

- name: Setup CLI dependencies
run: |
cd cli
./setup.sh

- name: Build CLI
run: |
cd cli
cargo build --release --target ${{ matrix.target }}

- name: Ad-hoc sign (macOS)
if: contains(matrix.os, 'macos')
run: codesign --force --sign - cli/target/${{ matrix.target }}/release/worldview

- name: Package
run: |
mkdir -p dist
cp cli/target/${{ matrix.target }}/release/worldview dist/worldview-${{ matrix.suffix }}
cd dist
tar -czvf worldview-${{ matrix.suffix }}.tar.gz worldview-${{ matrix.suffix }}

- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: binaries-${{ matrix.suffix }}
path: dist/*.tar.gz

release:
needs: build
runs-on: ubuntu-latest
permissions:
contents: write

steps:
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: artifacts

- name: Create Release
uses: softprops/action-gh-release@v2
with:
files: artifacts/**/*.tar.gz
generate_release_notes: true
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,4 @@ results/

# Generated files (created at compile-time)
validator/src/tokens_generated.rs
.codey/
79 changes: 41 additions & 38 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,53 +105,50 @@ Claims follow positional grammar—position implies role:

This repository includes a complete toolchain for working with Worldview documents:

### Validator (`worldview-validate`)
### CLI (`worldview`)

Validates `.wvf` files for syntactic correctness.
A unified command-line tool for working with Worldview files.

```bash
# Validate a file
worldview-validate example.wvf
worldview validate example.wvf

# Validate from stdin
cat example.wvf | worldview-validate --stdin
```

### Agent CLI (`worldview`)
cat example.wvf | worldview validate --stdin

An AI-powered tool that converts plain-text statements into proper Worldview notation.

```bash
# Add a fact to a Worldview file
worldview "Trust is built slowly through consistent actions" --file worldview.wvf
# Add a fact using AI agent
worldview add "Trust is built slowly through consistent actions" --file worldview.wvf

# Use a specific model
worldview "Power corrupts when unchecked" --model claude-opus-4-5-20251101
worldview add "Power corrupts when unchecked" --model claude-opus-4-5-20251101

# View format specification
worldview --help
```

The agent understands the full Worldview specification and will:
1. Read existing content to understand structure
2. Determine appropriate concept/facet placement
3. Format statements using proper notation
4. Validate before writing
The `add` command uses an AI agent that:
1. Reads existing content to understand structure
2. Determines appropriate concept/facet placement
3. Formats statements using proper notation
4. Validates before writing (validation runs automatically)

### Evaluation Framework

A Python framework for testing how well LLMs can leverage Worldview-encoded beliefs.

```bash
# Install dependencies
pip install -r evals/requirements.txt
uv sync

# Run evaluations
python -m evals run --models claude-sonnet gpt-5.2
uv run python -m evals run --models claude-sonnet gpt-5.2

# Run specific difficulty
python -m evals run --difficulty extreme
uv run python -m evals run --difficulty extreme

# List available models and test cases
python -m evals list-models
python -m evals list-cases
uv run python -m evals list-models
uv run python -m evals list-cases
```

## Project Structure
Expand All @@ -162,18 +159,17 @@ wvf/
│ ├── tokens.yaml # Token definitions (source of truth)
│ ├── grammar.pest # PEG grammar
│ └── generate.py # Generates docs and code from tokens.yaml
├── validator/ # Rust validator
│ ├── src/lib.rs # Validation library
│ ├── src/main.rs # CLI tool
├── validator/ # Rust validation library
│ ├── src/lib.rs # Validation logic
│ └── build.rs # Generates tokens from spec at compile time
├── agent/ # Rust agent CLI
│ └── src/main.rs # LLM-powered notation converter
├── cli/ # Rust CLI (unified binary)
│ ├── src/main.rs # Subcommand dispatch
│ ├── src/validate.rs # Validate subcommand
│ └── src/add.rs # Add subcommand (AI agent)
├── evals/ # Python evaluation framework
│ ├── cli.py # Evaluation CLI
│ ├── runner.py # Test orchestration
│ ├── evaluator.py # Response scoring
│ ├── test_cases.py # Test case definitions
│ └── llm_clients.py # LLM provider clients
│ ├── read_eval/ # Read comprehension tests
│ └── write_eval/ # Write/generation tests
├── SPEC.md # Full format specification
├── system.md # Condensed system prompt (generated)
└── example.wvf # Example document
Expand All @@ -184,16 +180,23 @@ wvf/
### Prerequisites

- Rust 1.70+
- Python 3.11+
- Python 3.11+ and [uv](https://docs.astral.sh/uv/)

### Build Tools
### Build CLI

```bash
# Build validator
cd validator && cargo build --release
# Setup vendored dependencies and build
cd cli && ./setup.sh && cargo build --release

# Build agent (requires setup for vendored dependencies)
cd agent && ./setup.sh && cargo build --release
# Binary will be at cli/target/release/worldview
```

### Run Evaluations

```bash
# From project root
uv sync
uv run python -m evals --help
```

## Design Principles
Expand Down
Loading