diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6cf73fab..85e487f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -73,7 +73,7 @@ jobs: # are heavy enough to exhaust the GitHub Actions CPU-minutes quota and # get killed mid-run. All of that coverage runs on the GPU cluster. uv run pytest tests/unit/ \ - -m "not requires_model and not gpu and not slow and not deep" \ + -m "not requires_model and not gpu and not slow and not deep and not release" \ -v -s --tb=short -x \ --cov=granite_switch --cov-report=xml diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index b5d0c5cf..6b642d86 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -60,11 +60,11 @@ on: # family is added HERE and nowhere else -- this entry plus its arm in the # Resolve test scope step below. suite: - description: "Which tests to run: full (all five suites), short (vllm + integration), dev (one fast GPU file), audio (the audio/ASR marker), multi (all five on 2 GPUs)." + description: "Which tests to run: full (all five suites), short (vllm + integration), dev (one fast GPU file), audio (the audio/ASR marker), multi (all five on 2 GPUs), release (the pre-release tier: real external models)." required: false default: full type: choice - options: [full, short, dev, audio, multi] + options: [full, short, dev, audio, multi, release] # RESULT CACHE OPT-OUT. Default off, i.e. cache on: the expensive thing here is # a GPU run, and re-running one for a commit+scope that was already tested buys # nothing in the common case (a reviewer re-reading a PR, a second maintainer @@ -225,19 +225,33 @@ jobs: # so a path list would be long and would drift silently every time a # marked test is added. A marker cannot drift. # - # "and not deep" is not optional. pytest's -m is single-valued and the - # command line REPLACES pyproject's addopts `-m "not deep"` rather than - # adding to it, so a bare `-m audio` would quietly start running the - # expensive deep audio tests that every other scope excludes. + # "and not deep and not release" is not optional. pytest's -m is + # single-valued and the command line REPLACES pyproject's addopts + # `-m "not deep and not release"` rather than adding to it, so a bare + # `-m audio` would quietly start running the expensive deep and + # release-tier audio tests that every other scope excludes. # # The quotes survive sed -> YAML -> helm -> the pod's shell; the # neighbouring `python -c "..."` lines in the job template rely on the # same thing. The one forbidden character is #, the sed delimiter. audio) - PATHS='-m "audio and not deep" tests/' + PATHS='-m "audio and not deep and not release" tests/' SUFFIX="audio" LEG="$LABEL-audio" ;; + # The pre-release tier, by MARKER for the same reason as `audio`: tests + # against real external artifacts (the default ASR, Hub repo layouts, + # quantization libraries) plus the non-representative cases of the + # breadth matrices. Every other scope excludes them. Run before tagging + # a release, and on PRs that bump uv.lock or a model pin. See + # docs/CICD.md, "Release test tier". + release) + # "and not deep": -m replaces addopts, so the deep code-theory + # suite would otherwise ride along on a release run. + PATHS='-m "release and not deep" tests/' + SUFFIX="release" + LEG="$LABEL-release" + ;; # Everything `full` runs, but on TWO GPUs. Not a different test # selection -- a different SHAPE of pod. # diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5ed9aa3..7751cb66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,10 +31,13 @@ repos: - id: check-case-conflict - id: mixed-line-ending args: [--fix=lf] + # tests/shared/fixtures/ holds byte-for-byte copies of published Hub files + # (chat templates, configs): a fixed-up final newline changes what a + # template renders, so the captures must stay verbatim. - id: end-of-file-fixer - exclude: \.ipynb$ + exclude: (\.ipynb$|^tests/shared/fixtures/) - id: trailing-whitespace - exclude: \.ipynb$ + exclude: (\.ipynb$|^tests/shared/fixtures/) - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.9.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index b5858986..0d426e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,34 @@ This project uses [semantic versioning](https://semver.org/). ## [Unreleased] +### Breaking + +- Audio checkpoints must name their ASR model. Serving no longer falls back to a + built-in default when `asr_model_id` is null; it fails at startup with the fix. + Checkpoints composed before this release need `asr_model_id` and + `asr_model_revision` added to `config.json`, or a re-compose (see + docs/AUDIO.md, *Migrating a checkpoint composed before 0.2.0*). +- Checkpoints composed with this release need this release to serve: older + versions ignore the new ASR fields and load the ASR unpinned from the Hub. +- `DEFAULT_ASR_MODEL_ID` moved from `granite_switch.vllm.audio` to + `granite_switch.config`. `ASRTranscriber` and `get_transcriber` now require + `model_id`. + +### Added + +- `--bundle-asr` composes the ASR model's weights into the checkpoint (`asr/`) + for airgapped serving with no Hub access; `asr_bundled_path` in the config. +- `--asr-revision` and the `asr_model_revision` config field. + +### Fixed + +- The default ASR model is pinned to its English-only Hub revision (`947f59a`). + It loaded unpinned, so the Hub repo's 2026-10-21 multilingual release silently + changed every transcript (now capitalized and punctuated) and broke the GPU + tests. +- GPU ASR tests no longer assert transcript text, which is model quality, not + something these tests own. + ## [0.1.0] — MVP Initial MVP release. API is not yet stable. diff --git a/docs/AUDIO.md b/docs/AUDIO.md index 4cca72bb..bb12f04c 100644 --- a/docs/AUDIO.md +++ b/docs/AUDIO.md @@ -50,13 +50,29 @@ This adds the `<|audio|>` marker token to the tokenizer and writes the audio settings into `config.json` so the checkpoint is self-describing: ```json -{ "asr_enabled": true, "asr_model_id": null, "asr_device": "cuda" } +{ "asr_enabled": true, + "asr_model_id": "ibm-granite/granite-speech-5.0-470m-turboctc", + "asr_model_revision": "947f59af40db9791170a0628cf0f3f4812d720f1", + "asr_bundled_path": null, + "asr_device": "cuda" } ``` -- `asr_model_id` — HF id of the speech-to-text model (default: - `ibm-granite/granite-speech-5.0-470m-turboctc`, a 470M English conformer CTC - encoder). Override with `--asr-model `, e.g. `openai/whisper-small` for - multilingual. +- `asr_model_id` — HF id of the speech-to-text model. Compose always writes it: + `ibm-granite/granite-speech-5.0-470m-turboctc` (a 470M English conformer CTC + encoder) unless `--asr-model ` is given, e.g. `openai/whisper-small` for + multilingual. Serving has **no built-in default**: an audio checkpoint without + `asr_model_id` is refused at startup, so the config on the Hub always shows + which ASR a checkpoint runs. +- `asr_model_revision` — Hub revision (commit, tag or branch) the ASR model loads + at. The default model is pinned to `947f59a`, its English-only release; the + repo's `main` moved on 2026-10-21 to a multilingual release that capitalizes and + punctuates, which silently changed every unpinned transcript. Set it with + `--asr-revision`. A custom `--asr-model` without one composes with a warning and + loads the repo's current `main` at serve time. A `revision` in + `asr_pipeline_kwargs` overrides it. +- `asr_bundled_path` — set by `--bundle-asr` (see *Airgapped deployment* below). + When set, the ASR loads from that directory inside the checkpoint and + `asr_model_id`/`asr_model_revision` only record where the copy came from. - `asr_device` — `cuda` (default): the default encoder is small and its speed comes from running on GPU. Set `--asr-device cpu` to leave vLLM's whole GPU memory budget to the KV cache — transcription is then several times slower @@ -85,6 +101,62 @@ options configure the cascade; they do not enable it. Passing one without deliberate: one explicit flag decides, rather than the decision being inferred from which options happen to be set. +### Airgapped deployment + +A checkpoint normally downloads its ASR model from the Hub on first use. For an +environment with no internet access, compose with `--bundle-asr` (it implies +`--enable-audio`): + +```bash +python -m granite_switch.composer.compose_granite_switch \ + --base-model ibm-granite/granite-4.1-3b \ + --built-in-adapters core \ + --bundle-asr \ + --output ./granite-switch-audio-bundled +``` + +This copies the ASR model, at its pinned revision, into `/asr/` (config, +safetensors weights, tokenizer, feature extractor, README and license; about +1 GB for the default model) and sets `"asr_bundled_path": "asr"`. Compose then +loads the bundle with `local_files_only=True`, so a bundle that would need the +network fails at compose time, not at serve time. Only safetensors models can be +bundled. + +`--asr-model` may also be a local directory, so the bundle can be built inside the +airgapped network from a copy of the ASR repo brought in by hand. A local +directory needs `--bundle-asr` (a plain `--enable-audio` checkpoint would point at +a path that exists on the build machine only), and the config records it by its +directory name, not its full path. + +Serve it with the Hub switched off: + +```bash +HF_HUB_OFFLINE=1 python -m vllm.entrypoints.openai.api_server \ + --model ./granite-switch-audio-bundled +``` + +A bundled checkpoint can also be uploaded to the Hub and served by repo id: only +its `asr/` folder is fetched for the ASR, through the normal HF cache, so a +pre-populated cache serves offline too. vLLM does not mistake `asr/*.safetensors` +for LLM weights: it only reads weight files at the checkpoint's top level. + +### Migrating a checkpoint composed before 0.2.0 + +Older audio checkpoints have `"asr_model_id": null` and relied on a default +inside the library, which no longer exists. Serving one fails at startup with an +error naming the fix. Either re-compose, or add these two fields to its +`config.json`: + +```json +"asr_model_id": "ibm-granite/granite-speech-5.0-470m-turboctc", +"asr_model_revision": "947f59af40db9791170a0628cf0f3f4812d720f1" +``` + +Checkpoints composed with 0.2.0 need granite-switch 0.2.0 or newer to serve +correctly. An older library ignores `asr_model_revision` and `asr_bundled_path` +without an error: it loads the ASR unpinned from the Hub, and a bundled +checkpoint still goes online. + ### Tuning the ASR model Two optional config fields let a checkpoint carry ASR tuning so no code change is @@ -319,8 +391,8 @@ fallback. `language` override is inert. For other languages use `--asr-model` with a multilingual generative model and set the language via `asr_generate_kwargs` (or per request via `mm_processor_kwargs`; see *Tuning the ASR model* above). -- **Transcripts from the CTC default are lowercase and unpunctuated** - (`what is the capital of israel`). They are spliced into the prompt as ordinary +- **Transcripts from the CTC default (at its pinned revision) are lowercase and + unpunctuated** (`what is the capital of israel`). They are spliced into the prompt as ordinary text, so the LLM reads them that way. A generative backend such as Whisper restores case and punctuation. - **HF `pipeline` backends only.** Any `automatic-speech-recognition` pipeline diff --git a/docs/CICD.md b/docs/CICD.md index a6805dd4..f68c2656 100644 --- a/docs/CICD.md +++ b/docs/CICD.md @@ -80,7 +80,7 @@ Two workflows run automatically when a PR is opened against `main` or a commit i **Jobs:** 1. `pre-commit` — runs `uvx pre-commit run --all-files`, executing the **exact same hooks** as the local pre-commit stage, pinned to the same versions in `.pre-commit-config.yaml`. This is the single source of truth: CI and local pre-commit can never drift, and any hook added or bumped in the config applies in CI automatically with no workflow edits. It covers ruff-format + ruff, `check-toml` / `check-yaml`, `validate-links` (broken local links / stale labels / broken first-party imports), `check-headers` (SPDX), the hygiene hooks (`end-of-file-fixer`, `trailing-whitespace`, `mixed-line-ending`, `check-merge-conflict`, `check-added-large-files`, `check-case-conflict`), `nbstripout`, and `uv-lock`. All are **blocking**: the tree is clean (see [Ruff rollout](#ruff-rollout-format-first-then-enforce)), so any new violation fails the build. Auto-fixing hooks (ruff, `end-of-file-fixer`, `check-headers`, `uv-lock`) modify files when something is wrong; pre-commit then reports failure and prints the diff (`--show-diff-on-failure`), telling the contributor to run pre-commit locally and re-commit. The DCO hooks (`add-signoff`, `check-dco`) run at the `prepare-commit-msg` / `commit-msg` stages, which `pre-commit run --all-files` does **not** execute — they are covered by `dco.yaml` instead. Hook environments are cached (`~/.cache/pre-commit`, keyed on the config file) so they don't reinstall every run. -2. `test-cpu` (needs `pre-commit`) — runs `tests/unit/` on Python 3.11 and 3.12 in parallel, filtered with `-m "not requires_model and not gpu and not slow and not deep"`. This is the **CI-safe set**: within `tests/unit/`, tests that download/compose real models (`requires_model`), need CUDA (`gpu`), hit the network or run long (`slow`), or are the expensive code-theory suite (`deep`) are still **excluded** by the marker filter. `tests/composer/` and `tests/hf/` are **not run in CI at all**: even filtered, they are heavy enough to exhaust the GitHub Actions CPU-minutes quota and get killed mid-run. All coverage beyond the unit suite — composer, HF, vLLM, and integration — runs on the GPU cluster instead (see [Manual: GPU Tests](#manual-gpu-tests)) +2. `test-cpu` (needs `pre-commit`) — runs `tests/unit/` on Python 3.11 and 3.12 in parallel, filtered with `-m "not requires_model and not gpu and not slow and not deep and not release"`. This is the **CI-safe set**: within `tests/unit/`, tests that download/compose real models (`requires_model`), need CUDA (`gpu`), hit the network or run long (`slow`), or are the expensive code-theory suite (`deep`) are still **excluded** by the marker filter. `tests/composer/` and `tests/hf/` are **not run in CI at all**: even filtered, they are heavy enough to exhaust the GitHub Actions CPU-minutes quota and get killed mid-run. All coverage beyond the unit suite — composer, HF, vLLM, and integration — runs on the GPU cluster instead (see [Manual: GPU Tests](#manual-gpu-tests)) Coverage is uploaded to [Codecov](https://codecov.io) after each test run (see [Coverage](#coverage) below). @@ -183,14 +183,87 @@ While in pre-1.0, increment `MINOR` for new features and `PATCH` for bug fixes. - Bump `version` in `pyproject.toml` - Add a section to `CHANGELOG.md` -2. **Merge to `main`** +2. **Run the release tier** — comment `/gpu-test-release` on the release PR and + get it green. It is the only place real external models are tested (see + [Release test tier](#release-test-tier)). -3. **Create a GitHub Release** (`Releases` → `Draft a new release`): +3. **Merge to `main`** + +4. **Create a GitHub Release** (`Releases` → `Draft a new release`): - Tag: `vX.Y.Z` (created from `main`) - Title: `vX.Y.Z` - Body: paste the `CHANGELOG.md` section for this release -4. **Click Publish release** — `publish.yaml` triggers automatically +5. **Click Publish release** — `publish.yaml` triggers automatically + +### Release test tier + +**PR runs download no real model.** Every test that needs a Granite base or an +adapter library runs twice: once on a fake on every PR run, and once on the real +model in the release tier, marked `release`. Release tests are excluded from every +other scope (`addopts` and each `-m` filter carry `not release`) and run with the +`release` GPU suite (`/gpu-test-release`). + +**The fakes are captured, not invented.** `tests/shared/fixtures/` holds verbatim +copies of the real repos' small files: each base's config, generation config, +tokenizer config, special-tokens map and chat template, and each adapter's +`adapter_config.json` and `io.yaml`. At test time, +`tests/shared/fake_bases.py` adds tiny random weights (in the real dtype) and a +tokenizer whose special tokens, and the words tests use as probe labels, keep +their real ids; `tests/shared/fake_adapter_libraries.py` adds synthetic weights at +each adapter's real rank and target modules. So the composer sees the real +template, token ids, configs and repo layout, and the real compose CLI accepts +the fakes. What the fakes leave out is weights and the BPE vocabulary: ordinary +words tokenize to one token per byte. + +**Writing a test that needs a model:** take it from `tests/shared/model_sources.py` +rather than naming a Hub id, and parametrize over `model_sources.sources()`: + +```python +@pytest.mark.parametrize("source", model_sources.sources()) +def test_something(source): + base = model_sources.base_model(source, "granite-4.1-3b") + library = model_sources.adapter_library( + source, "ibm-granite/granite-lib-rag-r1.0", "granite-4.1-3b" + ) +``` + +**Refreshing the captures:** `python -m tests.shared.fake_bases --capture` and +`python -m tests.shared.fake_adapter_libraries --capture`. A failing real case in +the release tier, whose fake case passes, usually means a capture has fallen +behind its Hub repo. + +**Release only, with no fake counterpart**, because the fake cannot stand in: + +- `tests/integration/test_asr_ctc_default_gpu.py`: the real default ASR at its + pinned revision. Every other audio test uses a tiny local ASR + (`tests/shared/tiny_asr.py`). +- `tests/vllm/test_generation_equivalence.py` and + `tests/composer/test_skinning_equivalence.py`: our model against upstream on the + real weights, with thresholds calibrated on them. On random weights the same + equivalence runs on PRs in `tests/vllm/test_granite4_mini.py`. +- `tests/composer/test_chat_template.py::TestInvocationFirstCharDropProperty`: + how the real BPE vocabulary splits invocation strings. +- `tests/composer/test_selective_download.py::TestRealHubMetadata`: the published + adapter repos' layout, against the live Hub. +- `tests/hf/test_quantization.py`: the quantization libraries on a real + checkpoint. Its `GRANITE_SWITCH_E2E_MODELS=1` gate is opened by + `tests/conftest.py` whenever `-m` selects `release`. + +**The rest of a breadth matrix** is release-only too: PR runs keep one +representative case. + +- `tests/vllm/test_granite4_mini.py`: vLLM classes on `4.0-micro`; `4.0-1b` and + `4.0-350m` in release. The CPU weight-transfer classes run every variant. +- `tests/vllm/test_granite4_fullsize.py`: the GPU class is release-only. +- `tests/vllm/test_multi_adapter_equivalence.py`: a fake `granite-4.1-3b` on PRs; + the real `4.1-3b` and `4.2-3b` (the ChatML template) in release. +- `tests/integration/test_audio_serving_smoke.py`: a fake `granite-4.0-micro` on + PRs; the real `4.0-micro` and `4.1-3b` in release. + +Run the release tier before every release, and on any PR that bumps `uv.lock`, +touches a model pin (e.g. `DEFAULT_ASR_MODEL_REVISION`) or a capture, or changes +code that depends on real weights or the BPE vocabulary. ### What `publish.yaml` does diff --git a/pyproject.toml b/pyproject.toml index ee201ae3..6a005b11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "granite-switch" -version = "0.1.1" +version = "0.2.0" description = "Granite Switch: Composable model building" readme = "README.md" license = "Apache-2.0" @@ -47,7 +47,7 @@ register_granite_switch = "granite_switch.vllm:register" [tool.pytest.ini_options] testpaths = ["tests"] -addopts = ["--ignore=tests/_legacy", "-m", "not deep"] +addopts = ["--ignore=tests/_legacy", "-m", "not deep and not release"] markers = [ "gpu: requires CUDA GPU", "vllm: requires vLLM installed", @@ -55,6 +55,7 @@ markers = [ "deep: expensive code-theory tests (m=8 / 256-dim); run with: pytest -m deep", "requires_model: needs a real model checkpoint", "audio: exercises the audio/ASR path; run with: pytest -m audio", + "release: pre-release tier, real external models (e.g. the default ASR) checked against our code; run with: pytest -m release", ] [dependency-groups] diff --git a/src/granite_switch/composer/README.md b/src/granite_switch/composer/README.md index 24c80b8f..71fb162f 100644 --- a/src/granite_switch/composer/README.md +++ b/src/granite_switch/composer/README.md @@ -216,9 +216,11 @@ This directory can be loaded directly: ```python # HuggingFace from granite_switch.hf import GraniteSwitchForCausalLM + model = GraniteSwitchForCausalLM.from_pretrained("./granite-with-all-aloras") # vLLM from vllm import LLM + llm = LLM(model="./granite-with-all-aloras") ``` diff --git a/src/granite_switch/composer/compose_granite_switch.py b/src/granite_switch/composer/compose_granite_switch.py index d7971982..583bdca2 100755 --- a/src/granite_switch/composer/compose_granite_switch.py +++ b/src/granite_switch/composer/compose_granite_switch.py @@ -53,6 +53,7 @@ import torch from huggingface_hub import snapshot_download +from huggingface_hub.utils import filter_repo_objects from transformers import AutoTokenizer from granite_switch.composer.adapter_discovery import ( @@ -83,7 +84,12 @@ ) from granite_switch.composer.validator import validate_control_lut from granite_switch.composer.weight_transfer import validate_untied_lm_head_saved -from granite_switch.config import ASR_DTYPES +from granite_switch.config import ( + ASR_BUNDLE_DIR, + ASR_DTYPES, + DEFAULT_ASR_MODEL_ID, + DEFAULT_ASR_MODEL_REVISION, +) from granite_switch.token_exchange import rebuild_control_to_substitute_lut # --------------------------------------------------------------------------- @@ -616,6 +622,130 @@ def _copy_upstream_auxiliary_files(base_model_local_path, output_path): return copied +def _resolve_asr_model(args) -> tuple[str, str | None]: + """``(asr_model_id, asr_model_revision)`` to write into an audio config. + + The default model is pinned to DEFAULT_ASR_MODEL_REVISION. A custom model + takes --asr-revision as given; without one it loads the repo's current main + at serve time, which can change under the checkpoint, so that is warned. + + A local directory is only a bundle source: without --bundle-asr serving + would load a path that exists on this machine alone, so it is refused. With + --bundle-asr the config records the directory's name, not its path, which + would leak the builder's filesystem into a published checkpoint. + """ + model_id = args.asr_model or DEFAULT_ASR_MODEL_ID + if os.path.isdir(model_id): + if not args.bundle_asr: + raise ValueError( + f"--asr-model {model_id} is a local directory, which only " + f"--bundle-asr can use: serving loads asr_model_id from the Hub, " + f"and this path exists on this machine alone. Add --bundle-asr, " + f"or pass a Hub repo id." + ) + return os.path.basename(os.path.normpath(model_id)), args.asr_revision + revision = args.asr_revision + if revision is None and model_id == DEFAULT_ASR_MODEL_ID: + revision = DEFAULT_ASR_MODEL_REVISION + if revision is None: + print( + f" WARNING: --asr-model {model_id} has no --asr-revision; it will " + f"load the repo's current main at serve time, so its transcripts can " + f"change without the checkpoint changing." + ) + return model_id, revision + + +# What a bundled ASR keeps from its source repo: everything transformers loads +# (config, weights, tokenizer, feature extractor) plus its README and license. +# Excludes legacy .bin/.pt weights and custom code, which a bundled model must +# not need. +_ASR_BUNDLE_PATTERNS = [ + "*.json", + "*.safetensors", + "*.txt", + "*.model", + "*.jinja", + "README.md", + "LICENSE*", +] + + +def _copy_asr_bundle_files(src: str, dest: str) -> None: + """Copy the files of a local ASR directory that a Hub bundle would get. + + Matches each file's path relative to ``src`` against _ASR_BUNDLE_PATTERNS + with the filter snapshot_download applies, so a local source and a Hub + source give the same bundle. Hidden directories (.git, .cache) are not + walked: a git-lfs clone would otherwise bring its object store along. + """ + rel_paths = [] + for dirpath, dirnames, filenames in os.walk(src): + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + rel_dir = os.path.relpath(dirpath, src) + rel_paths += [os.path.normpath(os.path.join(rel_dir, f)) for f in filenames] + for rel in filter_repo_objects(rel_paths, allow_patterns=_ASR_BUNDLE_PATTERNS): + os.makedirs(os.path.dirname(os.path.join(dest, rel)), exist_ok=True) + shutil.copy2(os.path.join(src, rel), os.path.join(dest, rel)) + + +def bundle_asr_model(model_id: str, revision: str | None, dest: str) -> None: + """Copy the ASR model into ``dest`` and check it loads with no network. + + ``model_id`` is a Hub repo (downloaded at ``revision``) or a local + directory (copied, so the bundle can be built inside an airgapped network). + """ + print(f"\nBundling ASR model {model_id} (revision {revision}) into {dest}") + if os.path.isdir(model_id) and os.path.realpath(model_id) == os.path.realpath(dest): + raise ValueError(f"--asr-model {model_id} is the bundle destination itself") + # Built and validated in a staging directory, then swapped in, so a failed + # download or validation leaves any existing bundle at dest untouched. + # Starting from an empty staging dir and replacing dest whole means a + # re-compose never leaves the previous ASR's files beside the new ones. + staging = f"{dest}.partial" + shutil.rmtree(staging, ignore_errors=True) + try: + if os.path.isdir(model_id): + _copy_asr_bundle_files(model_id, staging) + else: + snapshot_download( + repo_id=model_id, + revision=revision, + local_dir=staging, + allow_patterns=_ASR_BUNDLE_PATTERNS, + ) + # snapshot_download(local_dir=...) leaves its download cache metadata here. + shutil.rmtree(os.path.join(staging, ".cache"), ignore_errors=True) + _validate_asr_bundle(staging) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + shutil.rmtree(dest, ignore_errors=True) + os.replace(staging, dest) + size_gb, _ = _get_directory_size(dest) + if size_gb is not None: + print(f" ASR bundle size: {size_gb:.2f} GB") + + +def _validate_asr_bundle(path: str) -> None: + """Fail the compose if the bundle cannot load from disk alone.""" + if not os.path.isfile(os.path.join(path, "config.json")): + raise FileNotFoundError(f"Bundled ASR at {path} has no config.json") + if not any(f.endswith(".safetensors") for f in os.listdir(path)): + raise FileNotFoundError( + f"Bundled ASR at {path} has no .safetensors weights; only " + f"safetensors models can be bundled." + ) + from transformers import AutoConfig, AutoFeatureExtractor, AutoTokenizer + + # local_files_only proves the bundle is self-contained: nothing here may + # reach the Hub, exactly as at airgapped serve time. + AutoConfig.from_pretrained(path, local_files_only=True) + AutoFeatureExtractor.from_pretrained(path, local_files_only=True) + AutoTokenizer.from_pretrained(path, local_files_only=True) + print(" ASR bundle loads offline (config, feature extractor, tokenizer)") + + # Files that save_pretrained() is expected to write or overwrite. _EXPECTED_SAVE_FILES = { # model.save_pretrained @@ -837,9 +967,30 @@ def _compose_argparser(): "--asr-model", type=str, default=None, - help="HF id of the speech-to-text model the audio preprocessor loads. " - "Requires --enable-audio; ignored without it. Defaults to ibm-granite/" - "granite-speech-5.0-470m-turboctc (470M English CTC) when unset.", + help="HF id of the speech-to-text model the audio preprocessor loads, " + "written to asr_model_id. With --bundle-asr it may also be a local " + "directory, recorded in the config by its name only. Requires " + "--enable-audio; ignored without it. Defaults to " + f"{DEFAULT_ASR_MODEL_ID} (470M English CTC) when unset.", + ) + parser.add_argument( + "--asr-revision", + type=str, + default=None, + help="Hub revision (commit, tag or branch) of --asr-model, written to " + "asr_model_revision. Defaults to the pinned " + f"{DEFAULT_ASR_MODEL_REVISION[:7]} for the default model; a custom " + "--asr-model without it loads unpinned (warned). Requires " + "--enable-audio; ignored without it.", + ) + parser.add_argument( + "--bundle-asr", + action="store_true", + default=False, + help=f"Copy the ASR model's weights into /{ASR_BUNDLE_DIR}/ and " + "set asr_bundled_path, so the checkpoint serves audio with no Hub " + "access (airgapped deployments). Adds the ASR model's size (~1 GB for " + "the default) to the checkpoint. Implies --enable-audio.", ) parser.add_argument( "--asr-device", @@ -1178,7 +1329,11 @@ def build(): # default besides, so `--asr-device cuda:0` alone yielded a text-only # checkpoint with no diagnostic. A single explicit flag has no such gap, and # nothing to keep in step when an --asr-* option is added. - audio_enabled = args.enable_audio + audio_enabled = args.enable_audio or args.bundle_asr + # Resolved before the model is built, so a refused --asr-model fails fast. + asr_model_id, asr_revision = ( + _resolve_asr_model(args) if audio_enabled else (None, None) + ) # The control tokens are re-passed so this call doesn't drop them from the # tokenizer's additional-special-tokens list (add_special_tokens replaces # that list rather than extending it). @@ -1336,7 +1491,12 @@ def build(): # self-describing and the vLLM backend gates audio on asr_enabled. if audio_enabled: model.config.asr_enabled = True - model.config.asr_model_id = args.asr_model + # Always named in the config (serving has no default), so a checkpoint + # on the Hub shows which ASR it runs and at which revision. + model.config.asr_model_id = asr_model_id + model.config.asr_model_revision = asr_revision + if args.bundle_asr: + model.config.asr_bundled_path = ASR_BUNDLE_DIR model.config.asr_device = args.asr_device model.config.asr_dtype = args.asr_dtype # Optional pipeline-construction extras and default decode kwargs. Only @@ -1357,7 +1517,9 @@ def build(): model.config.asr_chunk_overlap_s = args.asr_chunk_overlap_s print( f" Audio cascade enabled " - f"(asr_model_id={args.asr_model or 'default'}, " + f"(asr_model_id={asr_model_id}, " + f"asr_model_revision={asr_revision or 'unpinned'}, " + f"bundled={args.bundle_asr}, " f"asr_device={args.asr_device}, " f"asr_dtype={args.asr_dtype or 'auto'}, " f"audio_token_id={audio_token_id}, " @@ -1514,6 +1676,18 @@ def save_and_validate_model_artifacts( # Snapshot directory state before save_pretrained before_snapshot = _snapshot_directory(args.output) + # Bundle before save_pretrained: the config written below points at the + # bundle, so a bundle that fails to download or validate must fail the + # compose before that config exists. + if getattr(model.config, "asr_bundled_path", None): + # args.asr_model, not asr_model_id: for a local source the config + # records only the directory's name (see _resolve_asr_model). + bundle_asr_model( + args.asr_model or model.config.asr_model_id, + model.config.asr_model_revision, + os.path.join(args.output, model.config.asr_bundled_path), + ) + model.save_pretrained(args.output, max_shard_size="5GB") print("Model saved") tokenizer.save_pretrained(args.output) diff --git a/src/granite_switch/composer/reporting/model_card.py b/src/granite_switch/composer/reporting/model_card.py index 13d7cbc0..7d0d9658 100644 --- a/src/granite_switch/composer/reporting/model_card.py +++ b/src/granite_switch/composer/reporting/model_card.py @@ -413,6 +413,10 @@ def _short_source(source): model.config, "adapter_substitute_token_ids", None ), "target_model": getattr(args, "target_model", None), + # Which ASR an audio checkpoint runs, and whether it is on disk. + "asr_model_id": getattr(model.config, "asr_model_id", None), + "asr_model_revision": getattr(model.config, "asr_model_revision", None), + "asr_bundled_path": getattr(model.config, "asr_bundled_path", None), } # Parameter counts: base is captured during transfer (see # weight_transfer.py); composed is summed from the in-memory model. diff --git a/src/granite_switch/config.py b/src/granite_switch/config.py index f5f358cb..64d3acdc 100644 --- a/src/granite_switch/config.py +++ b/src/granite_switch/config.py @@ -6,6 +6,22 @@ # Accepted asr_dtype values. Keep in sync with vllm.audio.asr._ASR_DTYPE_NAMES. ASR_DTYPES = ("auto", "float16", "bfloat16", "float32") +# Speech-to-text model the composer writes into an audio checkpoint when +# --asr-model is not given: Granite Speech 5.0 TurboCTC, a 470M English conformer +# CTC encoder. Serving has no fallback of its own; the config always names the +# model, so a checkpoint on the Hub shows which ASR it runs. +DEFAULT_ASR_MODEL_ID = "ibm-granite/granite-speech-5.0-470m-turboctc" + +# Hub commit DEFAULT_ASR_MODEL_ID is pinned to: its English-only release +# (lowercase, unpunctuated). The repo's main branch moved to a multilingual +# release on 2026-10-21 that capitalizes and punctuates, which silently changed +# every transcript of an unpinned checkpoint. +DEFAULT_ASR_MODEL_REVISION = "947f59af40db9791170a0628cf0f3f4812d720f1" + +# Subdirectory of the checkpoint that holds bundled ASR weights (compose +# --bundle-asr), for airgapped serving with no Hub access. +ASR_BUNDLE_DIR = "asr" + # Decoder-layer cache slots the switch reserves at the front of the model when # adapters are present. MultiSwitch (the only engine) owns two: a counting slot # and a memory slot. Single source of truth, mirrored by @@ -53,9 +69,20 @@ class GraniteSwitchConfig(GraniteMoeHybridConfig): Audio (ASR) preprocessing parameters (see docs/AUDIO.md): asr_enabled (bool): Register the audio preprocessor that transcribes audio and splices the transcript into the prompt. Default: False. - asr_model_id (Optional[str]): HF id of the speech-to-text model. None - falls back to the built-in default (Granite Speech 5.0 TurboCTC, - a 470M English CTC encoder). + asr_model_id (Optional[str]): HF id of the speech-to-text model. The + composer always sets it (DEFAULT_ASR_MODEL_ID unless --asr-model + is given); serving refuses an audio checkpoint without it. With + asr_bundled_path set it records where the bundled copy came from + (a local source by its directory name only). + asr_model_revision (Optional[str]): Hub revision (commit, tag or + branch) asr_model_id loads at. The composer pins the default model + to DEFAULT_ASR_MODEL_REVISION. None loads the repo's current main. + A "revision" in asr_pipeline_kwargs overrides it. Default: None. + asr_bundled_path (Optional[str]): Path of bundled ASR weights, + relative to the checkpoint directory (ASR_BUNDLE_DIR when composed + with --bundle-asr). When set, the ASR loads from there with no + network access and asr_model_id/asr_model_revision are not used + for loading. Default: None. asr_device (str): Device the ASR model runs on. Default "cuda" — the default encoder is small and GPU-bound work is what makes it fast. Set "cpu" to keep vLLM's GPU memory budget entirely for KV cache. @@ -171,6 +198,8 @@ def __init__( # Audio (ASR) preprocessing parameters asr_enabled: bool = False, asr_model_id: str | None = None, + asr_model_revision: str | None = None, + asr_bundled_path: str | None = None, asr_device: str = "cuda", asr_dtype: str | None = None, asr_pipeline_kwargs: dict | None = None, @@ -303,6 +332,8 @@ def __init__( # fields make the checkpoint self-describing about its ASR front-end. self.asr_enabled = asr_enabled self.asr_model_id = asr_model_id + self.asr_model_revision = asr_model_revision + self.asr_bundled_path = asr_bundled_path self.asr_device = asr_device # Validated here so a typo fails at compose time, not in a vLLM worker. if asr_dtype is not None and asr_dtype not in ASR_DTYPES: diff --git a/src/granite_switch/vllm/audio/__init__.py b/src/granite_switch/vllm/audio/__init__.py index feab7ce7..da4ad3fe 100644 --- a/src/granite_switch/vllm/audio/__init__.py +++ b/src/granite_switch/vllm/audio/__init__.py @@ -5,10 +5,9 @@ spliced into the prompt, so the decoder only ever sees text. See docs/AUDIO.md. """ -from .asr import DEFAULT_ASR_MODEL_ID, ASRTranscriber, transcribe +from .asr import ASRTranscriber, transcribe __all__ = [ - "DEFAULT_ASR_MODEL_ID", "ASRTranscriber", "transcribe", ] diff --git a/src/granite_switch/vllm/audio/asr.py b/src/granite_switch/vllm/audio/asr.py index 88eab82b..61e896a5 100644 --- a/src/granite_switch/vllm/audio/asr.py +++ b/src/granite_switch/vllm/audio/asr.py @@ -2,8 +2,12 @@ """Speech-to-text backend for the audio cascade. Wraps a HuggingFace ASR pipeline. Free of any vLLM import so it unit-tests on -CPU. The model loads lazily and is cached per (model_id, device, dtype, -pipeline_kwargs), so a process loads each ASR model at most once. +CPU. The model loads lazily and is cached per (model_id, revision, device, +dtype, pipeline_kwargs), so a process loads each ASR model at most once. + +There is no default model here: the checkpoint config names it (the composer +writes config.DEFAULT_ASR_MODEL_ID and its pinned revision), and ``model_id`` +may be a Hub id or a local directory (bundled weights). Device defaults to CUDA (the default CTC encoder is small and GPU-bound work is what makes it fast); dtype follows the device unless a checkpoint sets @@ -12,7 +16,9 @@ from __future__ import annotations +import functools import os +import posixpath import subprocess import threading from collections.abc import Mapping @@ -20,13 +26,7 @@ import numpy as np -# Default speech-to-text model: Granite Speech 5.0 TurboCTC, a 470M conformer -# CTC encoder. Used when the checkpoint does not name its own -# (config.asr_model_id is None). Non-autoregressive (one forward pass + greedy -# CTC collapse), so it cannot loop or hallucinate, but it also has no decoder to -# steer: output is lowercase and unpunctuated and language/task decode kwargs do -# not apply. English only. -DEFAULT_ASR_MODEL_ID = "ibm-granite/granite-speech-5.0-470m-turboctc" +from granite_switch.config import DEFAULT_ASR_MODEL_ID, DEFAULT_ASR_MODEL_REVISION # Call-time chunk window handed to a *generative* (seq2seq) pipeline, which has a # fixed input window and stitches its own chunks from timestamps. Never handed to @@ -169,6 +169,79 @@ def _unsupported_architecture_error(model_id: str, exc: Exception) -> Exception: ) +def _bundle_subdir(asr_bundled_path: str) -> str: + """``asr_bundled_path`` as a normalized subdirectory of the checkpoint. + + One spelling serves both the Hub fetch pattern and the on-disk path, and + the value comes from a config that may have been downloaded, so it must + not reach outside the checkpoint: absolute paths and ``..`` are refused. + """ + subdir = posixpath.normpath(asr_bundled_path.replace("\\", "/")) + if posixpath.isabs(subdir) or subdir == "." or subdir.split("/")[0] == "..": + raise ValueError( + f"asr_bundled_path={asr_bundled_path!r} must be a subdirectory of " + f"the checkpoint (compose with --bundle-asr writes 'asr')." + ) + return subdir + + +@functools.cache +def resolve_asr_source( + asr_model_id: str | None, + asr_model_revision: str | None, + asr_bundled_path: str | None, + checkpoint: str | None, + checkpoint_revision: str | None = None, +) -> tuple[str, str | None]: + """Return ``(model, revision)`` for the ASR pipeline from checkpoint config. + + Bundled weights win: ``asr_bundled_path`` (relative to the checkpoint) loads + from disk with no revision. ``checkpoint`` is the served model, a local + directory or a Hub repo id; for a repo id only the bundle subfolder is + fetched, which honours ``HF_HUB_OFFLINE`` and so resolves from the cache in + an airgapped deployment. Otherwise ``asr_model_id`` loads at + ``asr_model_revision``. There is no default: an audio checkpoint that names + neither is rejected, so the config always shows which ASR it runs. + + Memoized (every transcription resolves it); a failure is not cached. + """ + if asr_bundled_path: + if not checkpoint: + raise ValueError( + f"asr_bundled_path={asr_bundled_path!r} is set but the checkpoint " + f"location is unknown, so the bundled ASR cannot be found." + ) + subdir = _bundle_subdir(asr_bundled_path) + if os.path.isdir(checkpoint): + root = checkpoint + else: + from huggingface_hub import snapshot_download + + root = snapshot_download( + checkpoint, + revision=checkpoint_revision, + allow_patterns=[f"{subdir}/*"], + ) + path = os.path.join(root, subdir) + if not os.path.isfile(os.path.join(path, "config.json")): + raise FileNotFoundError( + f"Bundled ASR not found at {path!r} (asr_bundled_path=" + f"{asr_bundled_path!r}). Re-compose with --bundle-asr, or unset " + f"asr_bundled_path to load asr_model_id from the Hub." + ) + return path, None + if not asr_model_id: + raise ValueError( + "This checkpoint has asr_enabled but no asr_model_id. Serving no " + "longer falls back to a built-in ASR model; the config must name it. " + "Re-compose with --enable-audio, or add to the checkpoint's " + f'config.json: "asr_model_id": "{DEFAULT_ASR_MODEL_ID}", ' + f'"asr_model_revision": "{DEFAULT_ASR_MODEL_REVISION}". Checkpoints composed before granite-switch 0.2.0 ' + "need this. See docs/AUDIO.md." + ) + return asr_model_id, asr_model_revision + + _CHUNKING = None @@ -214,12 +287,14 @@ class ASRTranscriber: def __init__( self, - model_id: str = DEFAULT_ASR_MODEL_ID, + model_id: str, device: str = "cuda", pipeline_kwargs: Mapping[str, Any] | None = None, dtype: str | None = None, + revision: str | None = None, ) -> None: self.model_id = model_id + self.revision = revision self.device = device self.dtype = dtype self.pipeline_kwargs: dict[str, Any] = dict(pipeline_kwargs or {}) @@ -245,6 +320,8 @@ def load(self) -> None: "device": self.device, "torch_dtype": _resolve_torch_dtype(self.dtype, self.device), } + if self.revision is not None: + kwargs["revision"] = self.revision # pipeline_kwargs last: a checkpoint may override any default above. kwargs.update(self.pipeline_kwargs) try: @@ -368,28 +445,30 @@ def _freeze(value: Any) -> Any: def get_transcriber( - model_id: str | None = None, + model_id: str, device: str = "cuda", pipeline_kwargs: Mapping[str, Any] | None = None, dtype: str | None = None, + revision: str | None = None, ) -> ASRTranscriber: """Return a process-wide cached :class:`ASRTranscriber`. - Cached per ``(model_id, device, dtype, pipeline_kwargs)``. ``model_id`` of - None resolves to :data:`DEFAULT_ASR_MODEL_ID`. + Cached per ``(model_id, revision, device, dtype, pipeline_kwargs)``. """ - resolved = model_id or DEFAULT_ASR_MODEL_ID - key = (resolved, device, dtype, _freeze(pipeline_kwargs or {})) + if not model_id: + raise ValueError("get_transcriber requires a model_id") + key = (model_id, revision, device, dtype, _freeze(pipeline_kwargs or {})) transcriber = _TRANSCRIBERS.get(key) if transcriber is None: with _CACHE_LOCK: transcriber = _TRANSCRIBERS.get(key) if transcriber is None: transcriber = ASRTranscriber( - model_id=resolved, + model_id=model_id, device=device, pipeline_kwargs=pipeline_kwargs, dtype=dtype, + revision=revision, ) _TRANSCRIBERS[key] = transcriber return transcriber @@ -399,7 +478,8 @@ def transcribe( audio: AudioInput, sampling_rate: int | None = None, *, - model_id: str | None = None, + model_id: str, + revision: str | None = None, device: str = "cuda", pipeline_kwargs: Mapping[str, Any] | None = None, dtype: str | None = None, @@ -414,6 +494,7 @@ def transcribe( device=device, pipeline_kwargs=pipeline_kwargs, dtype=dtype, + revision=revision, ).transcribe( audio, sampling_rate, diff --git a/src/granite_switch/vllm/audio/processor.py b/src/granite_switch/vllm/audio/processor.py index 8cb16738..d1e43bf6 100644 --- a/src/granite_switch/vllm/audio/processor.py +++ b/src/granite_switch/vllm/audio/processor.py @@ -36,8 +36,8 @@ from .asr import ( DEFAULT_ALLOWED_REQUEST_GENERATE_KEYS, - DEFAULT_ASR_MODEL_ID, get_transcriber, + resolve_asr_source, resolve_driver_device, resolve_generate_kwargs, ) @@ -63,6 +63,10 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]: # No modalities on a non-audio checkpoint, so vLLM never loads ASR. if not self._asr_enabled(): return {} + # Called at engine startup: resolving here makes a checkpoint with no + # ASR model, or a missing bundle, fail before serving, not on the first + # audio request. + self._asr_source() # Finite so vLLM can size KV for the worst case. --limit-mm-per-prompt # may lower this ceiling, not raise it. return {"audio": self._asr_max_audio_clips()} @@ -85,9 +89,17 @@ def get_data_parser(self) -> MultiModalDataParser: # --- ASR config resolved from the model's GraniteSwitchConfig --- - def _asr_model_id(self) -> str: + def _asr_source(self) -> tuple[str, str | None]: + """``(model, revision)`` the ASR pipeline loads: bundled path or Hub id.""" cfg = self.get_hf_config() - return getattr(cfg, "asr_model_id", None) or DEFAULT_ASR_MODEL_ID + model_config = getattr(self.ctx, "model_config", None) + return resolve_asr_source( + asr_model_id=getattr(cfg, "asr_model_id", None), + asr_model_revision=getattr(cfg, "asr_model_revision", None), + asr_bundled_path=getattr(cfg, "asr_bundled_path", None), + checkpoint=getattr(model_config, "model", None), + checkpoint_revision=getattr(model_config, "revision", None), + ) def _asr_device(self) -> str: cfg = self.get_hf_config() @@ -271,8 +283,10 @@ def _transcribe( get :data:`_EMPTY_TRANSCRIPT_TEXT` instead, so the model sees an audio turn that simply said nothing. """ + model, revision = self.info._asr_source() transcriber = get_transcriber( - model_id=self.info._asr_model_id(), + model_id=model, + revision=revision, device=self.info._asr_device(), pipeline_kwargs=self.info._asr_pipeline_kwargs(), dtype=self.info._asr_dtype(), diff --git a/tests/composer/test_adapter_filtering.py b/tests/composer/test_adapter_filtering.py index d1b641c7..11d79000 100644 --- a/tests/composer/test_adapter_filtering.py +++ b/tests/composer/test_adapter_filtering.py @@ -11,6 +11,7 @@ list_available_adapters, ) from granite_switch.composer.arch import resolve_arch +from tests.shared import model_sources # -- Fixtures ---------------------------------------------------------------- @@ -58,7 +59,7 @@ def _make_adapter_library(tmp_path, target_model, adapters): class TestTechnologyFilterAdapters: def test_test_prefer_alora(self, tmp_path): # Load base config early for arch resolution. - arch = resolve_arch("ibm-granite/granite-4.0-micro") + arch = resolve_arch(model_sources.base_model("fake", "granite-4.0-micro")) root = _make_adapter_library( tmp_path, "granite-4.0-micro", @@ -97,7 +98,7 @@ def test_test_prefer_alora(self, tmp_path): assert len([found for found in adapters if found[2] == "lora"]) == 1 def test_filter_and_override(self, tmp_path): - arch = resolve_arch("ibm-granite/granite-4.0-micro") + arch = resolve_arch(model_sources.base_model("fake", "granite-4.0-micro")) root = _make_adapter_library( tmp_path, "granite-4.0-micro", diff --git a/tests/composer/test_adapter_loader.py b/tests/composer/test_adapter_loader.py index 9cb54254..b36a0b3c 100644 --- a/tests/composer/test_adapter_loader.py +++ b/tests/composer/test_adapter_loader.py @@ -20,6 +20,7 @@ load_adapter_target_modules, ) from granite_switch.composer.arch import ArchDescriptor, ModuleDescriptor, resolve_arch +from tests.shared import model_sources @pytest.fixture @@ -517,7 +518,7 @@ def test_fallback_precedence_and_yaml_parity(self, tmp_path): # --- ACTION: DISCOVERY MODE --- # input_path is the directory input_path = str(tmp_path) - arch = resolve_arch("ibm-granite/granite-4.0-micro") + arch = resolve_arch(model_sources.base_model("fake", "granite-4.0-micro")) adapters = discover_adapters( # default is to prefer alora input_path, "granite-4.0-micro", arch, technology_fallback="alora" ) diff --git a/tests/composer/test_asr_bundle.py b/tests/composer/test_asr_bundle.py new file mode 100644 index 00000000..841c5f6a --- /dev/null +++ b/tests/composer/test_asr_bundle.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Compose-side ASR selection and bundling (--asr-model/--asr-revision/--bundle-asr). + +No network: bundling is exercised from a tiny local ASR directory built here, +the same path an airgapped compose takes. +""" + +import pytest + +from granite_switch.composer.compose_granite_switch import ( + _compose_argparser, + _resolve_asr_model, + bundle_asr_model, +) +from granite_switch.config import DEFAULT_ASR_MODEL_ID, DEFAULT_ASR_MODEL_REVISION +from tests.shared.tiny_asr import build_tiny_asr + +pytestmark = pytest.mark.audio + + +def _args(*extra): + return _compose_argparser().parse_args(["--adapters", "x", *extra]) + + +class TestResolveAsrModel: + def test_default_model_is_pinned(self): + assert _resolve_asr_model(_args("--enable-audio")) == ( + DEFAULT_ASR_MODEL_ID, + DEFAULT_ASR_MODEL_REVISION, + ) + + def test_explicit_revision_wins_on_default_model(self): + args = _args("--enable-audio", "--asr-revision", "abc") + assert _resolve_asr_model(args) == (DEFAULT_ASR_MODEL_ID, "abc") + + def test_custom_model_without_revision_warns_and_is_unpinned(self, capsys): + args = _args("--enable-audio", "--asr-model", "openai/whisper-small") + assert _resolve_asr_model(args) == ("openai/whisper-small", None) + assert "no --asr-revision" in capsys.readouterr().out + + def test_custom_model_with_revision(self, capsys): + args = _args( + "--enable-audio", + "--asr-model", + "openai/whisper-small", + "--asr-revision", + "r", + ) + assert _resolve_asr_model(args) == ("openai/whisper-small", "r") + assert "WARNING" not in capsys.readouterr().out + + def test_local_dir_without_bundle_is_refused(self, tmp_path): + args = _args("--enable-audio", "--asr-model", str(tmp_path)) + with pytest.raises(ValueError, match="--bundle-asr"): + _resolve_asr_model(args) + + def test_local_dir_with_bundle_records_only_its_name(self, tmp_path, capsys): + src = tmp_path / "home" / "alice" / "my-asr" + src.mkdir(parents=True) + args = _args("--bundle-asr", "--asr-model", f"{src}/") + assert _resolve_asr_model(args) == ("my-asr", None) + # A bundle never loads from the Hub, so there is no unpinned warning. + assert "WARNING" not in capsys.readouterr().out + + def test_bundle_asr_is_opt_in(self): + assert _args("--enable-audio").bundle_asr is False + assert _args("--bundle-asr").bundle_asr is True + + +def _tiny_asr_dir(root): + """The shared tiny CTC model, plus a legacy weight file to be filtered.""" + build_tiny_asr(root) + (root / "pytorch_model.bin").write_bytes(b"legacy") + return root + + +class TestBundleAsrModel: + def test_local_source_is_copied_and_validated(self, tmp_path): + src = _tiny_asr_dir(tmp_path / "src") + dest = tmp_path / "ckpt" / "asr" + bundle_asr_model(str(src), None, str(dest)) + assert (dest / "config.json").is_file() + assert (dest / "model.safetensors").is_file() + # Legacy weights are not bundled. + assert not (dest / "pytorch_model.bin").exists() + + def test_source_without_safetensors_is_refused(self, tmp_path): + src = _tiny_asr_dir(tmp_path / "src") + (src / "model.safetensors").unlink() + with pytest.raises(FileNotFoundError, match="safetensors"): + bundle_asr_model(str(src), None, str(tmp_path / "asr")) + + def test_hub_source_downloads_at_the_revision(self, tmp_path, monkeypatch): + from granite_switch.composer import compose_granite_switch as cgs + + src = _tiny_asr_dir(tmp_path / "src") + calls = {} + + def fake_snapshot_download(repo_id, revision, local_dir, allow_patterns): + calls.update(repo_id=repo_id, revision=revision) + import shutil + + shutil.copytree(src, local_dir, dirs_exist_ok=True) + + monkeypatch.setattr(cgs, "snapshot_download", fake_snapshot_download) + bundle_asr_model("org/asr", "abc", str(tmp_path / "asr")) + assert calls == {"repo_id": "org/asr", "revision": "abc"} + + def test_rebundle_replaces_the_previous_bundle(self, tmp_path): + dest = tmp_path / "ckpt" / "asr" + dest.mkdir(parents=True) + # Left over from an earlier compose with another ASR model. + (dest / "model-00001-of-00002.safetensors").write_bytes(b"old") + (dest / "tokenizer.json").write_text("{}") + bundle_asr_model(str(_tiny_asr_dir(tmp_path / "src")), None, str(dest)) + assert not (dest / "model-00001-of-00002.safetensors").exists() + assert not (dest / "tokenizer.json").exists() + assert (dest / "model.safetensors").is_file() + + def test_source_that_is_the_destination_is_refused(self, tmp_path): + dest = _tiny_asr_dir(tmp_path / "asr") + with pytest.raises(ValueError, match="bundle destination"): + bundle_asr_model(str(dest), None, str(dest)) + assert (dest / "model.safetensors").is_file() + + def test_local_source_subdirectories_are_filtered_like_the_hub(self, tmp_path): + src = _tiny_asr_dir(tmp_path / "src") + # A git-lfs clone's object store, an ONNX export and a nested legacy + # weight file: none of them is in a Hub bundle. + (src / ".git" / "lfs").mkdir(parents=True) + (src / ".git" / "lfs" / "blob.json").write_text("{}") + (src / "onnx").mkdir() + (src / "onnx" / "model.onnx").write_bytes(b"onnx") + (src / "onnx" / "config.json").write_text("{}") + (src / "pt").mkdir() + (src / "pt" / "pytorch_model.bin").write_bytes(b"legacy") + dest = tmp_path / "asr" + bundle_asr_model(str(src), None, str(dest)) + assert not (dest / ".git").exists() + assert not (dest / "onnx" / "model.onnx").exists() + assert not (dest / "pt").exists() + # A nested file a pattern matches is kept, as snapshot_download keeps it. + assert (dest / "onnx" / "config.json").is_file() + + def test_failed_bundle_leaves_the_previous_bundle_in_place(self, tmp_path): + dest = tmp_path / "ckpt" / "asr" + bundle_asr_model(str(_tiny_asr_dir(tmp_path / "good")), None, str(dest)) + bad = _tiny_asr_dir(tmp_path / "bad") + (bad / "model.safetensors").unlink() + with pytest.raises(FileNotFoundError, match="safetensors"): + bundle_asr_model(str(bad), None, str(dest)) + assert (dest / "model.safetensors").is_file() + assert not (tmp_path / "ckpt" / "asr.partial").exists() + + +def test_failed_bundle_fails_the_compose_before_the_config_is_saved( + tmp_path, monkeypatch +): + """config.json points at the bundle, so it must not exist if bundling fails.""" + from types import SimpleNamespace + from unittest import mock + + from granite_switch.composer import compose_granite_switch as cgs + + monkeypatch.setattr(cgs, "_copy_io_configs", lambda *a, **k: {}) + monkeypatch.setattr(cgs, "_create_adapter_index", lambda *a, **k: {}) + + def failing_bundle(*args): + raise FileNotFoundError("no safetensors") + + monkeypatch.setattr(cgs, "bundle_asr_model", failing_bundle) + model = mock.MagicMock() + model.config.asr_bundled_path = "asr" + args = SimpleNamespace( + output=str(tmp_path / "out"), + create_ioyaml=False, + debug_fields=False, + base_model="base", + asr_model=None, + ) + base = tmp_path / "base" + base.mkdir() + with pytest.raises(FileNotFoundError): + cgs.save_and_validate_model_artifacts( + model=model, + tokenizer=mock.MagicMock(), + args=args, + base_model_local_path=str(base), + all_discovered=[], + adapter_token_ids=[], + ) + model.save_pretrained.assert_not_called() diff --git a/tests/composer/test_audio_marker_output_row.py b/tests/composer/test_audio_marker_output_row.py index 81ee15a4..de117977 100644 --- a/tests/composer/test_audio_marker_output_row.py +++ b/tests/composer/test_audio_marker_output_row.py @@ -35,6 +35,7 @@ ) from granite_switch.config import GraniteSwitchConfig from granite_switch.hf import GraniteSwitchForCausalLM +from tests.shared import model_sources # Ids inside _tiny_config's 300-token vocabulary. _AUDIO_ID = 252 @@ -109,13 +110,16 @@ def test_does_not_match_lookalike_tokens(self): ) assert find_reserved_never_emitted_token_id(tok) is None - def test_finds_reserved_token_on_real_granite_tokenizer(self): + @pytest.mark.parametrize("source", model_sources.sources()) + @pytest.mark.parametrize( + "base_name", ["granite-4.0-micro", "granite-4.1-3b", "granite-4.2-3b"] + ) + def test_finds_reserved_token_on_granite_tokenizer(self, source, base_name): + # fake: the captured tokenizer config (real added tokens on their real + # ids); real (release tier): the published tokenizer. from transformers import AutoTokenizer - try: - tok = AutoTokenizer.from_pretrained("ibm-granite/granite-4.1-3b") - except Exception as e: - pytest.skip(f"could not fetch Granite tokenizer: {e}") + tok = AutoTokenizer.from_pretrained(model_sources.base_model(source, base_name)) token_id = find_reserved_never_emitted_token_id(tok) assert token_id is not None assert tok.convert_ids_to_tokens(token_id).startswith("<|unused_") diff --git a/tests/composer/test_chat_template.py b/tests/composer/test_chat_template.py index 5cbabc0c..10965c92 100644 --- a/tests/composer/test_chat_template.py +++ b/tests/composer/test_chat_template.py @@ -267,6 +267,10 @@ def test_no_adapter_no_tokens(self): assert modified == original +# Release tier: the property is about how the real BPE vocabulary splits these +# strings, which is exactly what the fake bases leave out (one token per byte), so +# there is no fake counterpart that would test anything. +@pytest.mark.release class TestInvocationFirstCharDropProperty: """Standalone property test on a real Granite tokenizer for the invariant the Pass-2 edit relies on: dropping the first *token's worth of characters* @@ -294,12 +298,7 @@ class TestInvocationFirstCharDropProperty: def _get_tokenizer(self): from transformers import AutoTokenizer - try: - return AutoTokenizer.from_pretrained("ibm-granite/granite-4.1-3b") - except Exception as e: - import pytest - - pytest.skip(f"could not fetch Granite tokenizer: {e}") + return AutoTokenizer.from_pretrained("ibm-granite/granite-4.1-3b") def test_token_char_drop_equals_first_token_drop(self): """Dropping ``len(decode(first_token))`` characters from the string must diff --git a/tests/composer/test_compose_e2e.py b/tests/composer/test_compose_e2e.py index 30a1bd31..d26e9174 100644 --- a/tests/composer/test_compose_e2e.py +++ b/tests/composer/test_compose_e2e.py @@ -1,52 +1,81 @@ # SPDX-License-Identifier: Apache-2.0 """End-to-end compose test for ``python -m granite_switch.composer.compose_granite_switch``. -Runs the exact README command via subprocess, then performs sanity checks on the -output directory. The compose is expensive (~2.5 min, ~7 GB output) so a -module-scoped fixture composes once and shares the output across all test methods. - -Markers: slow, requires_model (both defined in pyproject.toml). +Runs the README command via subprocess, then performs sanity checks on the output +directory. A module-scoped fixture composes once per source and shares the output +across all test methods. + +Two sources (tests/shared/model_sources.py): ``fake`` on every PR run, a tiny base +and adapter library built from captured copies of the real ones; ``real``, the +published granite-4.0-micro and granite-lib-rag-r1.0, in the release tier. Expected +sizes are read off the base each source composes, so both run the same checks. """ import json +import math import subprocess import sys +from pathlib import Path +from types import SimpleNamespace import pytest -# --------------------------------------------------------------------------- -# Constants (verified against a known-good build of granite-4.0-micro + -# granite-lib-rag-r1.0) -# --------------------------------------------------------------------------- +from granite_switch.config import SWITCH_CACHE_LAYERS +from tests.shared import model_sources -BASE_MODEL = "ibm-granite/granite-4.0-micro" +BASE_NAME = "granite-4.0-micro" ADAPTER_LIBRARY = "ibm-granite/granite-lib-rag-r1.0" -BASE_VOCAB_SIZE = 100352 -BASE_NUM_LAYERS = 42 # switch model: base 40 + 2 MultiSwitch cache slots -BASE_PARAM_COUNT = 3_402_836_480 BUILD_TIMEOUT = 3600 # 60 min (parallel xdist workers compete for downloads) +def _base_facts(base): + """Vocab, hidden size, layer and parameter counts of the base being composed. + + Read from the base rather than hard-coded, so the fake and real sources run + the same checks. Parameter counts come from safetensors headers; no weights + are loaded. + """ + from huggingface_hub import get_safetensors_metadata + from safetensors import safe_open + from transformers import AutoConfig + + config = AutoConfig.from_pretrained(base) + if Path(base).is_dir(): + params = 0 + for sf in Path(base).glob("*.safetensors"): + with safe_open(str(sf), framework="pt") as f: + params += sum(math.prod(f.get_slice(k).get_shape()) for k in f.keys()) + else: + params = sum(get_safetensors_metadata(base).parameter_count.values()) + return SimpleNamespace( + vocab_size=config.vocab_size, + hidden_size=config.hidden_size, + num_layers=config.num_hidden_layers + SWITCH_CACHE_LAYERS, + param_count=params, + ) + + # --------------------------------------------------------------------------- # Fixture: run the build once, share output across all tests # --------------------------------------------------------------------------- -@pytest.fixture(scope="module") -def build_output(tmp_path_factory): - """Run the default build command and return the output directory Path. - - Uses ``tmp_path_factory`` so pytest manages cleanup. The fixture is - module-scoped so the ~2.5 min build happens only once. - """ - output_dir = tmp_path_factory.mktemp("build") / "granite-with-all-aloras" +@pytest.fixture(scope="module", params=model_sources.sources()) +def build(request, tmp_path_factory): + """Run the build once per source; returns the output dir and the base's facts.""" + source = request.param + base = model_sources.base_model(source, BASE_NAME) + library = model_sources.adapter_library(source, ADAPTER_LIBRARY, BASE_NAME) + output_dir = tmp_path_factory.mktemp(f"build-{source}") / "granite-switch" cmd = [ sys.executable, "-m", "granite_switch.composer.compose_granite_switch", + "--base-model", + base, "--adapters", - ADAPTER_LIBRARY, + library, "--output", str(output_dir), ] @@ -70,14 +99,19 @@ def build_output(tmp_path_factory): f"STDERR (last 1000 chars):\n{result.stderr[-1000:]}" ) - return output_dir + return SimpleNamespace(output=output_dir, base=_base_facts(base)) + + +@pytest.fixture(scope="module") +def build_output(build): + return build.output # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- -pytestmark = [pytest.mark.slow, pytest.mark.requires_model] +pytestmark = [pytest.mark.slow] @pytest.mark.xdist_group("compose_e2e") @@ -116,22 +150,22 @@ def test_output_files_exist(self, build_output): f"Expected at least 1 io.yaml file, found {len(yaml_files)}" ) - def test_config_correctness(self, build_output): + def test_config_correctness(self, build_output, build): """Verify key config.json fields match expected values.""" config = json.loads((build_output / "config.json").read_text()) assert config["model_type"] == "granite_switch" assert config["architectures"] == ["GraniteSwitchForCausalLM"] assert config["num_adapters"] >= 1 - assert config["hidden_size"] == 2560 - assert config["num_hidden_layers"] == BASE_NUM_LAYERS + assert config["hidden_size"] == build.base.hidden_size + assert config["num_hidden_layers"] == build.base.num_layers # Control token ID lists must match num_adapters num_adapters = config["num_adapters"] assert len(config["adapter_token_ids"]) == num_adapters - def test_parameter_count_increased(self, build_output): - """Verify total parameter count exceeds the base model's 3.4B.""" + def test_parameter_count_increased(self, build_output, build): + """Verify total parameter count exceeds the base model's.""" from safetensors import safe_open total_elements = 0 @@ -140,25 +174,27 @@ def test_parameter_count_increased(self, build_output): for key in f.keys(): total_elements += f.get_tensor(key).numel() - overhead = total_elements - BASE_PARAM_COUNT - overhead_pct = (overhead / BASE_PARAM_COUNT) * 100 + base_params = build.base.param_count + overhead = total_elements - base_params + overhead_pct = (overhead / base_params) * 100 print(f"\nParameter count: {total_elements:,}") - print(f"Base count: {BASE_PARAM_COUNT:,}") + print(f"Base count: {base_params:,}") print(f"Overhead: {overhead:,} ({overhead_pct:.2f}%)") - assert total_elements > BASE_PARAM_COUNT, ( - f"Expected > {BASE_PARAM_COUNT:,} params, got {total_elements:,}" + assert total_elements > base_params, ( + f"Expected > {base_params:,} params, got {total_elements:,}" ) - def test_vocabulary_expanded(self, build_output): + def test_vocabulary_expanded(self, build_output, build): """Verify vocab_size in config reflects the added control tokens.""" config = json.loads((build_output / "config.json").read_text()) num_adapters = config["num_adapters"] - expected_vocab = BASE_VOCAB_SIZE + num_adapters # 1 control token per adapter + base_vocab = build.base.vocab_size + expected_vocab = base_vocab + num_adapters # 1 control token per adapter assert config["vocab_size"] == expected_vocab, ( f"Expected vocab_size={expected_vocab} " - f"(base {BASE_VOCAB_SIZE} + {num_adapters}), " + f"(base {base_vocab} + {num_adapters}), " f"got {config['vocab_size']}" ) @@ -197,13 +233,13 @@ def test_compose_report_valid(self, build_output): assert key in report, f"Missing key '{key}' in compose_report.json" assert report[key], f"Empty value for '{key}' in compose_report.json" - def test_tokenizer_has_control_tokens(self, build_output): + def test_tokenizer_has_control_tokens(self, build_output, build): """Verify the saved tokenizer recognizes all control tokens.""" from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(str(build_output)) config = json.loads((build_output / "config.json").read_text()) - expected_vocab = BASE_VOCAB_SIZE + config["num_adapters"] + expected_vocab = build.base.vocab_size + config["num_adapters"] assert len(tokenizer) == expected_vocab, ( f"Expected tokenizer len={expected_vocab}, got {len(tokenizer)}" ) @@ -235,7 +271,7 @@ def test_generation_config_preserved(self, build_output): "generation_config.json missing bos_token_id" ) - def test_model_loads(self, build_output): + def test_model_loads(self, build_output, build): """Verify the model loads with device_map='meta' (no memory used).""" from transformers import AutoModelForCausalLM @@ -250,4 +286,4 @@ def test_model_loads(self, build_output): assert model is not None assert model.config.model_type == "granite_switch" assert model.config.num_adapters >= 1 - assert model.config.num_hidden_layers == BASE_NUM_LAYERS + assert model.config.num_hidden_layers == build.base.num_layers diff --git a/tests/composer/test_lora_substitute_probe.py b/tests/composer/test_lora_substitute_probe.py index 33aae30a..b62d4bf0 100644 --- a/tests/composer/test_lora_substitute_probe.py +++ b/tests/composer/test_lora_substitute_probe.py @@ -5,9 +5,9 @@ template rather than hardcoding a Granite-4.x-specific token string. These tests verify: -1. On real Granite tokenizers, the probe returns <|start_of_role|> (id - 100264) — the token the LoRA prefix insertion places immediately after - the control token in the rendered prompt. +1. On the Granite tokenizers, the probe returns the role marker the LoRA prefix + insertion places immediately after the control token in the rendered prompt: + <|start_of_role|> (id 100264) on 4.0/4.1, <|im_start|> (id 100256) on 4.2. 2. On a synthetic tokenizer with a different template, the probe returns whatever that template emits first for a user turn. 3. The probe raises a clear error when the template is missing, fails to @@ -22,46 +22,36 @@ from granite_switch.composer.compose_granite_switch import ( _probe_lora_substitute_token_id, ) +from tests.shared import model_sources _FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") -class TestOnRealGraniteTokenizer: - """Exercise the probe on actual Granite tokenizers. Network-dependent; - skips cleanly if the model can't be fetched.""" - - def _tok(self, name): +class TestOnGraniteTokenizer: + """The probe on each Granite base's tokenizer and chat template. + + ``fake`` (every PR run) is the captured tokenizer config and template on a tiny + vocabulary, whose special tokens keep their real ids; ``real`` (release tier) + is the published tokenizer. Same assertions on both. + """ + + @pytest.mark.parametrize("source", model_sources.sources()) + @pytest.mark.parametrize( + "base_name, token, token_id", + [ + ("granite-4.0-micro", "<|start_of_role|>", 100264), + ("granite-4.1-3b", "<|start_of_role|>", 100264), + # ChatML: the role marker is <|im_start|>. + ("granite-4.2-3b", "<|im_start|>", 100256), + ], + ) + def test_probe_returns_the_role_marker(self, source, base_name, token, token_id): from transformers import AutoTokenizer - try: - return AutoTokenizer.from_pretrained(name) - except Exception as e: - pytest.skip(f"could not fetch tokenizer {name!r}: {e}") - - def test_granite_4_1_3b(self): - tok = self._tok("ibm-granite/granite-4.1-3b") - sub_id = _probe_lora_substitute_token_id(tok) - assert sub_id == 100264 - assert tok.convert_ids_to_tokens([sub_id])[0] == "<|start_of_role|>" - - def test_granite_4_0_micro(self): - tok = self._tok("ibm-granite/granite-4.0-micro") - sub_id = _probe_lora_substitute_token_id(tok) - assert sub_id == 100264 - assert tok.convert_ids_to_tokens([sub_id])[0] == "<|start_of_role|>" - - def test_granite_4_2_chatml(self): - """Granite 4.2 (ChatML): the probe returns <|im_start|> (id 100256). - - The 4.2 base ships on a local path (env GRANITE_4_2_BASE), not the Hub; - skips when unavailable so the suite still runs locally. - """ - path = os.environ.get("GRANITE_4_2_BASE") - if not path or not os.path.exists(path): - pytest.skip("GRANITE_4_2_BASE not set / path missing") - tok = self._tok(path) + tok = AutoTokenizer.from_pretrained(model_sources.base_model(source, base_name)) sub_id = _probe_lora_substitute_token_id(tok) - assert tok.convert_ids_to_tokens([sub_id])[0] == "<|im_start|>" + assert sub_id == token_id + assert tok.convert_ids_to_tokens([sub_id])[0] == token class TestChatMLMultiShapeConstancy: @@ -70,8 +60,8 @@ class TestChatMLMultiShapeConstancy: the granite-format one (thinking on/off, system/no-system, tools/no-tools). Uses the ChatML fixture with a minimal fake tokenizer that records the - first token of each rendered shape and asserts they all agree. On Vela the - real-tokenizer test above pins the concrete id (<|im_start|>).""" + first token of each rendered shape and asserts they all agree. + TestOnGraniteTokenizer above pins the concrete id (<|im_start|>).""" def test_input_ids_0_constant_across_chatml_shapes(self): from jinja2 import Environment diff --git a/tests/composer/test_multi_audio_compose_e2e.py b/tests/composer/test_multi_audio_compose_e2e.py index a2b1de5e..05505789 100644 --- a/tests/composer/test_multi_audio_compose_e2e.py +++ b/tests/composer/test_multi_audio_compose_e2e.py @@ -13,13 +13,15 @@ one past the last control id, which makes the table stale and sends compose into the rebuild that used to be missing on ``MultiSwitch``. -Markers are ``slow`` + ``requires_model`` + ``audio``, matching -``test_compose_e2e.py`` and deliberately NOT env-gated: gating is what hid the -bug. ``tests/composer/test_control_lut_refresh.py`` covers the same call site in -seconds on a tiny model; this is the end-to-end proof that a real compose of the +Markers are ``slow`` + ``audio``, matching ``test_compose_e2e.py`` and +deliberately NOT env-gated: gating is what hid the bug. +``tests/composer/test_control_lut_refresh.py`` covers the same call site in +seconds on a tiny model; this is the end-to-end proof that a compose of the combination completes and ships a loadable checkpoint. -Cost: one compose of the default base model (module-scoped). +Two sources (tests/shared/model_sources.py): ``fake`` on every PR run, a tiny +granite-4.1-3b and adapter library built from captured copies of the real ones; +``real``, the published ones, in the release tier. """ import json @@ -28,22 +30,27 @@ import pytest -pytestmark = [pytest.mark.slow, pytest.mark.requires_model, pytest.mark.audio] +from tests.shared import model_sources -# Same adapter library as test_compose_e2e.py, with the default base model. +pytestmark = [pytest.mark.slow, pytest.mark.audio] + +# Same adapter library as test_compose_e2e.py, on the CLI's default base. # --technology-filter lora keeps the build to the cheapest useful shape. +BASE_NAME = "granite-4.1-3b" ADAPTER_LIBRARY = "ibm-granite/granite-lib-rag-r1.0" BUILD_TIMEOUT = 3600 _LUT_KEY = "model.switch.control_to_substitute_lut" -def _compose(output_dir): +def _compose(source, output_dir): cmd = [ sys.executable, "-m", "granite_switch.composer.compose_granite_switch", + "--base-model", + model_sources.base_model(source, BASE_NAME), "--adapters", - ADAPTER_LIBRARY, + model_sources.adapter_library(source, ADAPTER_LIBRARY, BASE_NAME), "--technology-filter", "lora", "--enable-audio", @@ -66,9 +73,10 @@ def _compose(output_dir): return output_dir -@pytest.fixture(scope="module") -def audio_multi(tmp_path_factory): - return _compose(tmp_path_factory.mktemp("audio-multi") / "model") +@pytest.fixture(scope="module", params=model_sources.sources()) +def audio_multi(request, tmp_path_factory): + source = request.param + return _compose(source, tmp_path_factory.mktemp(f"audio-multi-{source}") / "model") def _lut_numel(output_dir): @@ -161,19 +169,22 @@ def test_multi_audio_checkpoint_loads_without_size_overrides(audio_multi): # marker to its head. The audio marker grows the vocabulary past the control # ids, so this also checks the saved control LUT matches the shipped config. # --------------------------------------------------------------------------- # -_PROBE_BASE = "ibm-granite/granite-4.1-3b" _PROBE_LABELS = ["safe", "unsafe"] +#: The LoRA the manifest pairs with the probe head. +_PROBE_LORA = "citations" -def _write_probe_manifest(root): - """A real LoRA plus a synthetic bias-only head (label 1 always wins).""" +def _write_probe_manifest(source, root): + """A LoRA plus a synthetic bias-only head (label 1 always wins).""" import torch from safetensors.torch import save_file from transformers import AutoConfig - from .test_probe_compose import _find_rag_lora_dir - - hidden = AutoConfig.from_pretrained(_PROBE_BASE).hidden_size + base = model_sources.base_model(source, BASE_NAME) + lora_dir = model_sources.lora_adapter_dir( + source, ADAPTER_LIBRARY, BASE_NAME, _PROBE_LORA + ) + hidden = AutoConfig.from_pretrained(base).hidden_size head_dir = root / "safety" / "safety" / "granite-4.1-3b" / "lora" head_dir.mkdir(parents=True) bias = torch.tensor([-10.0, 10.0]) @@ -185,7 +196,7 @@ def _write_probe_manifest(root): manifest = root / "adapters.yaml" manifest.write_text( "rag:\n" - f" path: {_find_rag_lora_dir()}\n" + f" path: {lora_dir}\n" " type: lora\n" "safety:\n" f" path: {head_dir}\n" @@ -196,15 +207,15 @@ def _write_probe_manifest(root): return manifest -def _compose_probe(root, enable_audio): - manifest = _write_probe_manifest(root) +def _compose_probe(source, root, enable_audio): + manifest = _write_probe_manifest(source, root) output_dir = root / "model" cmd = [ sys.executable, "-m", "granite_switch.composer.compose_granite_switch", "--base-model", - _PROBE_BASE, + model_sources.base_model(source, BASE_NAME), "--adapters", str(manifest), "--output", @@ -221,10 +232,18 @@ def _compose_probe(root, enable_audio): return output_dir -@pytest.fixture(scope="module", params=[True, False], ids=["audio", "text"]) +@pytest.fixture( + scope="module", + params=[ + pytest.param((audio, src.values[0]), id=f"{name}-{src.id}", marks=src.marks) + for audio, name in ((True, "audio"), (False, "text")) + for src in model_sources.sources() + ], +) def probe_checkpoint(request, tmp_path_factory): - root = tmp_path_factory.mktemp("probe-audio" if request.param else "probe") - return request.param, _compose_probe(root, enable_audio=request.param) + enable_audio, source = request.param + root = tmp_path_factory.mktemp(f"probe-{source}{'-audio' if enable_audio else ''}") + return enable_audio, _compose_probe(source, root, enable_audio=enable_audio) @pytest.mark.xdist_group("multi_audio_compose_e2e") @@ -275,3 +294,13 @@ def _assert_lut_matches(output_dir, vocab_size): assert f.get_slice(_LUT_KEY).get_shape()[0] == vocab_size return pytest.fail(f"{_LUT_KEY} not found in {output_dir}") + + +def test_multi_audio_config_names_the_pinned_asr(audio_multi): + # Serving has no fallback ASR, so compose must write the model and revision. + from granite_switch.config import DEFAULT_ASR_MODEL_ID, DEFAULT_ASR_MODEL_REVISION + + config = json.loads((audio_multi / "config.json").read_text()) + assert config["asr_model_id"] == DEFAULT_ASR_MODEL_ID + assert config["asr_model_revision"] == DEFAULT_ASR_MODEL_REVISION + assert config.get("asr_bundled_path") is None diff --git a/tests/composer/test_probe_compose.py b/tests/composer/test_probe_compose.py index 0af60d29..6f199068 100644 --- a/tests/composer/test_probe_compose.py +++ b/tests/composer/test_probe_compose.py @@ -13,8 +13,8 @@ ``probe_head.safetensors`` into bank row ``slot = adapter_index - 1``. These tests build the model through :class:`GraniteSwitchComposer` (per CLAUDE.md -rule #5 — never hand-assemble the config) against the real ``granite-4.0-micro`` -base, with a *synthetic* probe head whose weights make one label win +rule #5 — never hand-assemble the config) against the ``granite-4.0-micro`` base, +with a *synthetic* probe head whose weights make one label win deterministically, and assert: 1. ``probe_head.weight[slot, :n_labels]`` / ``.bias[slot, :n_labels]`` exactly @@ -28,11 +28,11 @@ 4. Two probe slots with DIFFERENT label counts (2 and 3) each land padded correctly and each emit a token id from their own label set. -Marked slow + requires_model: composing the ~3B micro base is expensive, but it is -the only sanctioned construction path. +Two sources (tests/shared/model_sources.py): ``fake`` on every PR run, tiny bases +and a LoRA built from captured copies of the real ones; ``real``, the published +ones, in the release tier. """ -import glob import json import os @@ -42,36 +42,35 @@ import granite_switch.hf # noqa: F401 — registers AutoModel/AutoConfig from granite_switch.composer.compose_utils import GraniteSwitchComposer from granite_switch.composer.weight_transfer import PROBE_HEAD_FILE +from tests.shared import model_sources -BASE_MODEL = "ibm-granite/granite-4.0-micro" -# The mixed test embeds a real LoRA, which must match its base. The RAG library -# ships granite-4.1-3b LoRAs, so the mixed case composes against that base. -MIXED_BASE = "ibm-granite/granite-4.1-3b" -MIXED_LORA_TARGET = "granite-4.1-3b" +BASE_NAME = "granite-4.0-micro" +# The mixed test embeds a LoRA from the RAG library, which must match its base. +MIXED_BASE_NAME = "granite-4.1-3b" RAG_LIBRARY = "ibm-granite/granite-lib-rag-r1.0" +MIXED_LORA = "citations" -pytestmark = [pytest.mark.slow, pytest.mark.requires_model] +pytestmark = [ + pytest.mark.slow, + pytest.mark.parametrize("source", model_sources.sources()), +] # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -def _tokenizer(model=BASE_MODEL): +def _tokenizer(source, base=BASE_NAME): from transformers import AutoTokenizer - try: - return AutoTokenizer.from_pretrained(model) - except Exception as e: # pragma: no cover - network/env dependent - pytest.skip(f"could not fetch tokenizer {model!r}: {e}") + return AutoTokenizer.from_pretrained(model_sources.base_model(source, base)) -def _hidden_size(model=BASE_MODEL): +def _hidden_size(source, base=BASE_NAME): from transformers import AutoConfig - try: - return AutoConfig.from_pretrained(model).hidden_size - except Exception as e: # pragma: no cover - pytest.skip(f"could not fetch config {model!r}: {e}") + return AutoConfig.from_pretrained( + model_sources.base_model(source, base) + ).hidden_size def _single_token_labels(tokenizer, candidates, count=2): @@ -133,49 +132,13 @@ def _probe_prompt_ids(tokenizer, control_token_id): return torch.tensor([ids], dtype=torch.long) -def _find_rag_lora_dir(): - """Locate a real cached LoRA adapter dir for the mixed-case test. - - Reads the local HF cache directly (via ``scan_cache_dir``) rather than - ``snapshot_download``: the RAG library is large and only partially cached - here, and ``snapshot_download(local_files_only=True)`` refuses an incomplete - snapshot even when the one adapter this test needs is present. Skip cleanly - if no matching adapter is cached. - """ - from huggingface_hub import scan_cache_dir - - snapshot_dirs = [] - try: - for repo in scan_cache_dir().repos: - if repo.repo_id == RAG_LIBRARY: - snapshot_dirs.extend(str(rev.snapshot_path) for rev in repo.revisions) - except Exception as e: # pragma: no cover - pytest.skip(f"could not scan HF cache for {RAG_LIBRARY!r}: {e}") - - for root in snapshot_dirs: - hits = glob.glob( - os.path.join( - root, "**", MIXED_LORA_TARGET, "lora", "adapter_model.safetensors" - ), - recursive=True, - ) - # Follow the cache symlink to the real blob; skip broken (missing) ones. - hits = [h for h in sorted(hits) if os.path.exists(os.path.realpath(h))] - if hits: - return os.path.dirname(hits[0]) - - pytest.skip( - f"no cached {MIXED_LORA_TARGET} LoRA adapter with weights in {RAG_LIBRARY!r}" - ) - - # --------------------------------------------------------------------------- # # Tests # --------------------------------------------------------------------------- # -def test_probe_only_compose_loads_trained_head(tmp_path): +def test_probe_only_compose_loads_trained_head(source, tmp_path): """A single probe slot: weights land in the bank and the verdict fires.""" - tokenizer = _tokenizer() - hidden = _hidden_size() + tokenizer = _tokenizer(source) + hidden = _hidden_size(source) label_names, label_ids = _single_token_labels( tokenizer, ["safe", "unsafe", "yes", "no", "true", "false"] @@ -193,7 +156,7 @@ def test_probe_only_compose_loads_trained_head(tmp_path): control_id = tokenizer.vocab_size - 5 model = GraniteSwitchComposer.from_base_and_adapters( - base_model_name_or_path=BASE_MODEL, + base_model_name_or_path=model_sources.base_model(source, BASE_NAME), adapter_paths=[clf_dir], adapter_token_ids=[control_id], adapter_substitute_token_ids=[control_id], # inert for a probe slot @@ -228,11 +191,13 @@ def test_probe_only_compose_loads_trained_head(tmp_path): ) -def test_mixed_lora_and_probe_preserves_slot_numbering(tmp_path): +def test_mixed_lora_and_probe_preserves_slot_numbering(source, tmp_path): """[lora, probe]: LoRA slot 0 populated, probe slot 1 loaded.""" - tokenizer = _tokenizer(MIXED_BASE) - hidden = _hidden_size(MIXED_BASE) - lora_dir = _find_rag_lora_dir() + tokenizer = _tokenizer(source, MIXED_BASE_NAME) + hidden = _hidden_size(source, MIXED_BASE_NAME) + lora_dir = model_sources.lora_adapter_dir( + source, RAG_LIBRARY, MIXED_BASE_NAME, MIXED_LORA + ) label_names, label_ids = _single_token_labels( tokenizer, ["safe", "unsafe", "yes", "no", "true", "false"] @@ -248,7 +213,7 @@ def test_mixed_lora_and_probe_preserves_slot_numbering(tmp_path): clf_ctrl = tokenizer.vocab_size - 5 model = GraniteSwitchComposer.from_base_and_adapters( - base_model_name_or_path=MIXED_BASE, + base_model_name_or_path=model_sources.base_model(source, MIXED_BASE_NAME), adapter_paths=[lora_dir, clf_dir], adapter_token_ids=[lora_ctrl, clf_ctrl], adapter_substitute_token_ids=[lora_ctrl, clf_ctrl], @@ -309,7 +274,7 @@ def test_mixed_lora_and_probe_preserves_slot_numbering(tmp_path): assert torch.isfinite(out.logits[0, -1]).all() -def test_ragged_label_counts_pad_and_fire_per_slot(tmp_path): +def test_ragged_label_counts_pad_and_fire_per_slot(source, tmp_path): """Two probe slots with DIFFERENT label counts (2 and 3). Each head lands in the top rows of its (padded) bank row, the pad rows above @@ -317,8 +282,8 @@ def test_ragged_label_counts_pad_and_fire_per_slot(tmp_path): token emits a token id from THAT slot's label set — proving the padded einsum/scatter never leaks a phantom (padded) label across slots. """ - tokenizer = _tokenizer() - hidden = _hidden_size() + tokenizer = _tokenizer(source) + hidden = _hidden_size(source) # Slot A: 2 labels; slot B: 3 labels. Distinct token ids across both sets. names_a, ids_a = _single_token_labels( @@ -345,7 +310,7 @@ def test_ragged_label_counts_pad_and_fire_per_slot(tmp_path): ctrl_b = tokenizer.vocab_size - 6 model = GraniteSwitchComposer.from_base_and_adapters( - base_model_name_or_path=BASE_MODEL, + base_model_name_or_path=model_sources.base_model(source, BASE_NAME), adapter_paths=[dir_a, dir_b], adapter_token_ids=[ctrl_a, ctrl_b], adapter_substitute_token_ids=[ctrl_a, ctrl_b], @@ -417,12 +382,12 @@ def _first_lora_b_tensor(model): @pytest.mark.parametrize( "base", [ - "ibm-granite/granite-4.0-micro", - "ibm-granite/granite-4.1-3b", - "ibm-granite/granite-4.2-3b", # ChatML family + "granite-4.0-micro", + "granite-4.1-3b", + "granite-4.2-3b", # ChatML family ], ) -def test_probe_drops_generation_prompt(base): +def test_probe_drops_generation_prompt(base, source): """A probe render ignores ``add_generation_prompt=True``. The verdict is read at the marker, so the assistant generation prompt must @@ -434,7 +399,7 @@ def test_probe_drops_generation_prompt(base): configure_chat_template, ) - tokenizer = _tokenizer(base) + tokenizer = _tokenizer(source, base) discovered = [(None, "safety", "probe", None)] add_control_tokens(tokenizer, discovered) configure_chat_template(tokenizer, discovered) @@ -453,12 +418,12 @@ def render(gen, **kwargs): @pytest.mark.parametrize( "base", [ - "ibm-granite/granite-4.0-micro", - "ibm-granite/granite-4.1-3b", - "ibm-granite/granite-4.2-3b", # ChatML family + "granite-4.0-micro", + "granite-4.1-3b", + "granite-4.2-3b", # ChatML family ], ) -def test_probe_marker_is_last_token(base): +def test_probe_marker_is_last_token(base, source): """The marker is the very last token, after the last turn's close. Everything before it is byte-identical to a no-adapter render without a @@ -469,7 +434,7 @@ def test_probe_marker_is_last_token(base): configure_chat_template, ) - tokenizer = _tokenizer(base) + tokenizer = _tokenizer(source, base) discovered = [(None, "safety", "probe", None)] (control_id,), _ = add_control_tokens(tokenizer, discovered) configure_chat_template(tokenizer, discovered) @@ -496,17 +461,17 @@ def test_probe_marker_is_last_token(base): @pytest.mark.parametrize( "base, role_open", [ - ("ibm-granite/granite-4.0-micro", "<|start_of_role|>"), - ("ibm-granite/granite-4.1-3b", "<|start_of_role|>"), - ("ibm-granite/granite-4.2-3b", "<|im_start|>"), + ("granite-4.0-micro", "<|start_of_role|>"), + ("granite-4.1-3b", "<|start_of_role|>"), + ("granite-4.2-3b", "<|im_start|>"), ], ) -def test_probe_substitute_is_the_bases_role_open_marker(base, role_open): +def test_probe_substitute_is_the_bases_role_open_marker(base, role_open, source): from granite_switch.composer.compose_granite_switch import ( build_control_token_lists, ) - tokenizer = _tokenizer(base) + tokenizer = _tokenizer(source, base) discovered = [(None, "safety", "probe", None)] _, _, substitutes = build_control_token_lists( tokenizer, discovered, base_reset=False diff --git a/tests/composer/test_save_load_compose.py b/tests/composer/test_save_load_compose.py index 03af91e8..98d5dfd9 100644 --- a/tests/composer/test_save_load_compose.py +++ b/tests/composer/test_save_load_compose.py @@ -15,7 +15,10 @@ Load from Phase 1 output → save to second dir → load again. Compare the two loaded models AND the two directory contents. -Requires base model + RAG adapters download. Marked slow + requires_model. +Two sources (tests/shared/model_sources.py): ``fake`` on every PR run, a tiny base +and adapter library built from captured copies of the real ones; ``real``, the +published granite-4.1-3b and granite-lib-rag-r1.0, in the release tier. Every check +compares a model with itself across a save/load, so both run the same assertions. """ import filecmp @@ -31,10 +34,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer import granite_switch.hf # noqa: F401 — registers AutoModel +from tests.shared import model_sources -# Whole module downloads a base model + RAG adapters and composes a checkpoint — -# far too heavy for GitHub-hosted CPU runners; runs on the GPU cluster only. -pytestmark = [pytest.mark.slow, pytest.mark.requires_model] +# Composes and serializes a checkpoint twice: too heavy for GitHub-hosted CPU +# runners even on the fake source; runs on the GPU cluster only. +pytestmark = [pytest.mark.slow] + +BASE_NAME = "granite-4.1-3b" +ADAPTER_LIBRARY = "ibm-granite/granite-lib-rag-r1.0" SEED = 42 @@ -93,14 +100,16 @@ def _forward_logits(model, input_ids): return model(input_ids=input_ids, use_cache=False).logits.cpu() -def _call_build(output_dir, extra_args=None): +def _call_build(source, output_dir, extra_args=None): """Call compose_granite_switch.build() with mocked sys.argv.""" from granite_switch.composer.compose_granite_switch import build fake_argv = [ "compose_granite_switch", + "--base-model", + model_sources.base_model(source, BASE_NAME), "--adapters", - "ibm-granite/granite-lib-rag-r1.0", + model_sources.adapter_library(source, ADAPTER_LIBRARY, BASE_NAME), "--output", str(output_dir), ] @@ -203,8 +212,13 @@ def _make_golden_set(adapter_name: str) -> list[str]: # ════════════════════════════════════════════════════════════════════ +@pytest.fixture(scope="module", params=model_sources.sources()) +def source(request): + return request.param + + @pytest.fixture(scope="module") -def phase1(tmp_path_factory): +def phase1(source, tmp_path_factory): """build() → save_and_validate_model_artifacts() → from_pretrained(). Returns everything needed to compare built vs loaded: @@ -213,7 +227,7 @@ def phase1(tmp_path_factory): save_dir = str(tmp_path_factory.mktemp("phase1") / "model") # ── build() ── - build_result = _call_build(save_dir) + build_result = _call_build(source, save_dir) ( model, tokenizer, @@ -332,14 +346,14 @@ def phase1(tmp_path_factory): @pytest.fixture(scope="module") -def phase2(tmp_path_factory): +def phase2(source, tmp_path_factory): """build()+save → load → save to second dir → load again. Returns everything needed to compare the two loads and two directories. """ # ── First save: build() + save ── save_dir_1 = str(tmp_path_factory.mktemp("phase2a") / "model") - build_result = _call_build(save_dir_1) + build_result = _call_build(source, save_dir_1) adapter_token_ids = build_result[7] # index 7 = adapter_token_ids _call_save(build_result) del build_result diff --git a/tests/composer/test_selective_download.py b/tests/composer/test_selective_download.py index 64a95f7f..706a533b 100644 --- a/tests/composer/test_selective_download.py +++ b/tests/composer/test_selective_download.py @@ -314,6 +314,7 @@ def test_shared_include_filter_across_repos_downloads_disjoint_subsets( @pytest.mark.slow +@pytest.mark.release # pins the published repo's layout, which only the Hub changes class TestRealHubMetadata: REPO = "ibm-granite/granitelib-core-r1.0" TARGET_MODEL = "granite-4.1-3b" diff --git a/tests/composer/test_skinning_equivalence.py b/tests/composer/test_skinning_equivalence.py index 9be11867..eabbe612 100644 --- a/tests/composer/test_skinning_equivalence.py +++ b/tests/composer/test_skinning_equivalence.py @@ -17,7 +17,10 @@ - Fast (requires_model only): single 64-token request per model. - Thorough (slow + requires_model): 8 requests, lengths 3–256 tokens. -Markers: slow, requires_model (consistent with test_build_e2e.py). +Markers: slow, requires_model (consistent with test_build_e2e.py), and release: +the comparison is our model against upstream on the real weights, whose +tolerances were calibrated on them. On random weights the same equivalence runs +on every PR in tests/vllm/test_granite4_mini.py. """ import os @@ -28,6 +31,8 @@ import pytest +pytestmark = pytest.mark.release + HF_WORKER = Path(__file__).parent / "_skinning_equivalence_worker.py" VLLM_WORKER = Path(__file__).parent / "_skinning_equivalence_worker_vllm.py" HF_TIMEOUT = 1800 # 30 min per model (download + 2× load + forward) diff --git a/tests/composer/test_upstream_files.py b/tests/composer/test_upstream_files.py index 5a46a95a..91b55970 100644 --- a/tests/composer/test_upstream_files.py +++ b/tests/composer/test_upstream_files.py @@ -12,8 +12,11 @@ Tests the full pipeline including ``save_pretrained()`` overwrite behavior, model loading, and tokenizer. -Markers: requires_model (all tests need HF model downloads). - slow (full build tests only). +Two sources (tests/shared/model_sources.py): ``fake`` on every PR run, a tiny base +built from the captured files of the real repo (including the files the copy must +skip); ``real``, the published granite-4.0-micro, in the release tier. + +Markers: slow (full build tests only). """ import json @@ -27,19 +30,30 @@ _copy_upstream_auxiliary_files, _resolve_base_model_path, ) +from tests.shared import model_sources # --------------------------------------------------------------------------- # Model lists (Granite only) # --------------------------------------------------------------------------- +#: (family, base name in tests/shared/fake_bases.py). Smallest representative. FAMILY_MODELS = [ - ("granitemoehybrid", "ibm-granite/granite-4.0-micro"), + ("granitemoehybrid", "granite-4.0-micro"), ] -# Smallest representative per family for full build E2E. -BUILD_MODELS = [ - ("granitemoehybrid", "ibm-granite/granite-4.0-micro"), -] + +def _cases(models): + """Each (family, base) once per source; the real source is release-tier.""" + return [ + pytest.param( + (family, base, src.values[0]), + id=f"{family}-{src.values[0]}", + marks=src.marks, + ) + for family, base in models + for src in model_sources.sources() + ] + BUILD_TIMEOUT = 3600 # 60 min (parallel xdist workers compete for downloads) @@ -52,25 +66,27 @@ # --------------------------------------------------------------------------- -@pytest.fixture(scope="module", params=FAMILY_MODELS, ids=[f[0] for f in FAMILY_MODELS]) +@pytest.fixture(scope="module", params=_cases(FAMILY_MODELS)) def resolved_model(request): - """Resolve each family's model to a local path (downloads if needed). + """Resolve each base to a local path (downloads the real one if needed). Returns ``(family, model_id, local_path)``. """ - family, model_id = request.param + family, base, source = request.param + model_id = model_sources.base_model(source, base) local_path = _resolve_base_model_path(model_id) return family, model_id, local_path -@pytest.fixture(scope="module", params=BUILD_MODELS, ids=[b[0] for b in BUILD_MODELS]) +@pytest.fixture(scope="module", params=_cases(FAMILY_MODELS)) def built_in_build_output(request, tmp_path_factory): - """Run a built-in-only build for each BUILD_MODELS entry. + """Run a built-in-only build for each base and source. Returns ``(family, model_id, output_dir)``. """ - family, model_id = request.param - output_dir = tmp_path_factory.mktemp(f"build_{family}") / "output" + family, base, source = request.param + model_id = model_sources.base_model(source, base) + output_dir = tmp_path_factory.mktemp(f"build_{family}_{source}") / "output" cmd = [ sys.executable, @@ -109,8 +125,6 @@ def built_in_build_output(request, tmp_path_factory): # Copy logic tests # --------------------------------------------------------------------------- -pytestmark = pytest.mark.requires_model - class TestUpstreamFileCopy: """Verify _copy_upstream_auxiliary_files() across Granite model families.""" diff --git a/tests/conftest.py b/tests/conftest.py index e15f0b2c..c9014cda 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,6 +43,16 @@ def pytest_configure(config): # a newer toolkit in the image; this keeps the suites meaningful until then. os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") + # ── release tier opens the real-checkpoint gate ─────────────── + # + # Some release-tier tests (test_quantization.py) are also gated on + # GRANITE_SWITCH_E2E_MODELS=1, which the GPU pod has no way to set: the runner + # passes only pytest arguments. A run that selects the release marker has asked + # for exactly those tests, so it sets the gate itself. setdefault, so an explicit + # GRANITE_SWITCH_E2E_MODELS=0 still wins. + if "release" in (config.getoption("markexpr", "") or "").replace("not release", ""): + os.environ.setdefault("GRANITE_SWITCH_E2E_MODELS", "1") + worker_id = os.environ.get("PYTEST_XDIST_WORKER") if worker_id is None: return # not running under xdist diff --git a/tests/hf/test_quantization.py b/tests/hf/test_quantization.py index e4a21c98..86067638 100644 --- a/tests/hf/test_quantization.py +++ b/tests/hf/test_quantization.py @@ -32,6 +32,9 @@ pytest.mark.slow, pytest.mark.requires_model, pytest.mark.gpu, + # Compatibility with the quantization libraries on a real checkpoint; the + # release suite sets GRANITE_SWITCH_E2E_MODELS=1 so the gate below opens. + pytest.mark.release, pytest.mark.skipif( os.environ.get("GRANITE_SWITCH_E2E_MODELS") != "1", reason="composes a real ~3B mixed checkpoint; set GRANITE_SWITCH_E2E_MODELS=1", diff --git a/tests/integration/test_adapter_routing_audio_enabled.py b/tests/integration/test_adapter_routing_audio_enabled.py index 0eae2b0b..f966aa09 100644 --- a/tests/integration/test_adapter_routing_audio_enabled.py +++ b/tests/integration/test_adapter_routing_audio_enabled.py @@ -4,36 +4,45 @@ Asserts the switch maps ``adapter_token_ids[i]`` to index ``i + 1`` and leaves pre-control positions on base (``0``) — enabling audio must not perturb the -token->index map. For granite-4.1-3b, 12 of the 13 defined adapters resolve -(context_relevance ships no 4.1-3b flavor). +token->index map. -Markers: slow + requires_model + gpu (opt-in via -m). +Routing reads only the control-token table, never adapter weights, so the +default case composes empty built-in adapter slots (``--built-in-adapters``) +rather than real adapter libraries. Thirteen slots keep the sweep as wide as the +full granitelib set. Two sources (tests/shared/model_sources.py) for the base: +``fake`` on every PR run, ``real`` granite-4.0-micro in the release tier. The HF +model never loads the ASR, so no ASR is downloaded either. Real libraries can +still be swept through ``GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS``. + +Markers: slow + gpu (opt-in via -m). """ import importlib.util import json import os +from pathlib import Path import pytest pytestmark = [ pytest.mark.audio, pytest.mark.slow, - pytest.mark.requires_model, pytest.mark.gpu, ] if importlib.util.find_spec("granite_switch.hf") is None: pytest.skip("requires the HF backend ([hf] extra)", allow_module_level=True) +from tests.shared import model_sources -_DEFAULT_ADAPTER_LIBRARIES = [ - "ibm-granite/granitelib-rag-r1.0", - "ibm-granite/granitelib-core-r1.0", - "ibm-granite/granitelib-guardian-r1.0", -] +_BUILT_IN_ADAPTERS = [f"slot{i}" for i in range(1, 14)] +_BASE_NAME = "granite-4.0-micro" +# (source, base, adapter libraries); None means the built-in slots above. _DEFAULT_BASE_MODEL_PAIRS = [ - ("ibm-granite/granite-4.1-3b", _DEFAULT_ADAPTER_LIBRARIES), + pytest.param(("fake", _BASE_NAME, None), id=f"{_BASE_NAME}-fake"), + pytest.param( + ("real", _BASE_NAME, None), marks=pytest.mark.release, id=f"{_BASE_NAME}-real" + ), ] @@ -52,8 +61,9 @@ def _load_experimental_pairs(): pairs = [] for p in entries: adapters = p["adapter"] + adapters = adapters if isinstance(adapters, list) else [adapters] pairs.append( - (p["base"], adapters if isinstance(adapters, list) else [adapters]) + pytest.param(("path", p["base"], adapters), id=Path(p["base"]).name) ) return pairs @@ -66,11 +76,7 @@ def _load_experimental_pairs(): _FILLER_TOKENS = [791, 5679, 2766, 279, 893, 389, 813, 1450] -@pytest.fixture( - scope="module", - params=BASE_MODEL_PAIRS, - ids=lambda p: p[0].rsplit("/", 1)[-1], -) +@pytest.fixture(scope="module", params=BASE_MODEL_PAIRS) def audio_switch_model(request, tmp_path_factory): import subprocess import sys @@ -79,17 +85,22 @@ def audio_switch_model(request, tmp_path_factory): from granite_switch.hf import GraniteSwitchForCausalLM - base_model, adapter_libraries = request.param - save_dir = tmp_path_factory.mktemp(base_model.rsplit("/", 1)[-1]) / "model" + source, base, adapter_libraries = request.param + base_model = base if source == "path" else model_sources.base_model(source, base) + save_dir = tmp_path_factory.mktemp(Path(base_model).name) / "model" + adapter_args = ( + ["--built-in-adapters", *_BUILT_IN_ADAPTERS] + if adapter_libraries is None + else ["--adapters", *adapter_libraries] + ) cmd = [ sys.executable, "-m", "granite_switch.composer.compose_granite_switch", "--base-model", base_model, - "--adapters", - *adapter_libraries, + *adapter_args, "--enable-audio", "--output", str(save_dir), diff --git a/tests/integration/test_asr_ctc_default_gpu.py b/tests/integration/test_asr_ctc_default_gpu.py index e87a828a..7dc37f21 100644 --- a/tests/integration/test_asr_ctc_default_gpu.py +++ b/tests/integration/test_asr_ctc_default_gpu.py @@ -5,10 +5,17 @@ the path a served checkpoint takes rather than transformers in isolation: the default device, the dtype auto-resolution, the CTC classification that suppresses pipeline chunking and decode kwargs, and the window that decides whether a clip -reaches the model whole or through our chunker. +reaches the model whole or through our chunker. Loads the model at the revision +the composer pins (config.DEFAULT_ASR_MODEL_REVISION). -Downloads the ~1GB checkpoint on first run. Opt in explicitly: -`pytest -m "audio and requires_model and gpu"`. +Transcript text is deliberately not asserted: that is model quality, which these +tests do not own (the Hub release of this model changed its casing and +punctuation once already). They check that each path runs without raising. + +Release tier: the only test that downloads and runs the real default ASR +(~1GB); every other audio test uses tests/shared/tiny_asr.py. It guards what a +fake cannot: that the pinned model still loads and runs with our dependency +versions. Run with `pytest -m release` (`/gpu-test-release`). """ import importlib.util @@ -18,11 +25,14 @@ import numpy as np import pytest +from granite_switch.config import DEFAULT_ASR_MODEL_ID, DEFAULT_ASR_MODEL_REVISION + pytestmark = [ pytest.mark.audio, pytest.mark.slow, pytest.mark.requires_model, pytest.mark.gpu, + pytest.mark.release, ] torch = pytest.importorskip("torch") @@ -39,9 +49,6 @@ asr = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(asr) -# tests/audio/test1.wav says this, and CTC emits it lowercase and unpunctuated. -_EXPECTED = "what is the capital of israel" - def _test_clip(): """The committed test waveform as float32 mono, plus its sample rate.""" @@ -57,7 +64,11 @@ def _test_clip(): @pytest.fixture(scope="module") def transcriber(): """The default model on CUDA with the default (auto -> bfloat16) dtype.""" - instance = asr.ASRTranscriber(model_id=asr.DEFAULT_ASR_MODEL_ID, device="cuda") + instance = asr.ASRTranscriber( + model_id=DEFAULT_ASR_MODEL_ID, + revision=DEFAULT_ASR_MODEL_REVISION, + device="cuda", + ) instance.load() return instance @@ -72,30 +83,34 @@ def test_backend_is_detected_as_ctc(transcriber): assert transcriber._is_ctc is True -def test_transcribes_correctly_and_ignores_decode_kwargs(transcriber): +def test_decode_kwargs_are_dropped_not_forwarded(transcriber): samples, sample_rate = _test_clip() # `language` is what a client may send via mm_processor_kwargs. A CTC backend - # has no generate() to take it, so it must be dropped rather than forwarded. + # has no generate() to take it, so it must be dropped rather than forwarded, + # which would raise a TypeError. text = transcriber.transcribe( samples, sampling_rate=sample_rate, generate_kwargs={"language": "fr"} ) - assert text == _EXPECTED + assert isinstance(text, str) -def test_float16_override_still_transcribes(): +def test_float16_override_runs(): """float16 is an available override, not a failure mode. The default resolves to bfloat16 on CUDA because that is the checkpoint's own dtype and it keeps float32's exponent range next to this encoder's BatchNorm layers. An earlier version of this test asserted float16 *raised*; measured on an A100 (torch 2.10 / transformers 5.16) it does not, so what is worth - guarding is that the override keeps working and still produces the transcript. + guarding is that the override keeps loading and running. """ samples, sample_rate = _test_clip() half = asr.ASRTranscriber( - model_id=asr.DEFAULT_ASR_MODEL_ID, device="cuda", dtype="float16" + model_id=DEFAULT_ASR_MODEL_ID, + revision=DEFAULT_ASR_MODEL_REVISION, + device="cuda", + dtype="float16", ) - assert half.transcribe(samples, sampling_rate=sample_rate) == _EXPECTED + assert isinstance(half.transcribe(samples, sampling_rate=sample_rate), str) def _tile_to_seconds(samples, sample_rate, seconds): @@ -126,7 +141,7 @@ def counting(segment, generate_kwargs=None): transcriber._run_pipeline = counting try: - text = transcriber.transcribe(audio, sampling_rate=sample_rate) + transcriber.transcribe(audio, sampling_rate=sample_rate) finally: transcriber._run_pipeline = inner @@ -136,5 +151,3 @@ def counting(segment, generate_kwargs=None): assert max(calls) <= asr.DEFAULT_CHUNK_LENGTH_S + 1 else: assert len(calls) == expected_calls - # The phrase repeats throughout, so the merge must not collapse it to nothing. - assert _EXPECTED.split()[-1] in text diff --git a/tests/integration/test_audio_serving_smoke.py b/tests/integration/test_audio_serving_smoke.py index 11edcc3a..a4c0c5d3 100644 --- a/tests/integration/test_audio_serving_smoke.py +++ b/tests/integration/test_audio_serving_smoke.py @@ -5,6 +5,13 @@ path the low-level tests bypass (CUDA graphs, the ASR processor inside vLLM's EngineCore subprocess, the multi-clip splice, generation). +Composed with ``--bundle-asr`` (which implies ``--enable-audio``) from a tiny +random CTC model built locally (``tests/shared/tiny_asr.py``), so the ASR is +served from the checkpoint's ``asr/`` folder with no ASR download. This also +covers the airgapped path (issue #158) and proves vLLM does not take +``asr/*.safetensors`` for LLM weights. The real default ASR is exercised by +``test_asr_ctc_default_gpu.py``. + Deliberately does NOT assert transcript content: WER and adapter-routing correctness are separate boxes, covered by the eval harness and the MultiSwitch serving e2e tests. Synthetic tones keep the test asset-free. @@ -15,24 +22,46 @@ import importlib.util import json import os +from pathlib import Path import pytest pytestmark = [ pytest.mark.audio, pytest.mark.slow, - pytest.mark.requires_model, pytest.mark.gpu, ] if importlib.util.find_spec("vllm") is None: pytest.skip("requires vLLM installed", allow_module_level=True) +from tests.shared import model_sources + +# E2E model matrix: (base, adapter library). PR runs serve a fake granite-4.0-micro +# (tests/shared/model_sources.py); the release tier serves the real 4.0-micro and +# adds the real 4.1-3b as a second base. +_ADAPTER_LIBRARY = "ibm-granite/granitelib-core-r1.0" + + +def _pair(source, base_name): + return ( + model_sources.base_model(source, base_name), + model_sources.adapter_library(source, _ADAPTER_LIBRARY, base_name), + ) + -# E2E model matrix: base model + adapter-library pairs. _DEFAULT_BASE_MODEL_PAIRS = [ - ("ibm-granite/granite-4.0-micro", "ibm-granite/granitelib-core-r1.0"), - ("ibm-granite/granite-4.1-3b", "ibm-granite/granitelib-core-r1.0"), + pytest.param(("fake", "granite-4.0-micro"), id="granite-4.0-micro-fake"), + pytest.param( + ("real", "granite-4.0-micro"), + marks=pytest.mark.release, + id="granite-4.0-micro-real", + ), + pytest.param( + ("real", "granite-4.1-3b"), + marks=pytest.mark.release, + id="granite-4.1-3b-real", + ), ] @@ -52,7 +81,12 @@ def _load_experimental_pairs(): f"GRANITE_SWITCH_EXPERIMENTAL_MODEL_PAIRS is not valid JSON: {e}\n" f'Expected format: \'[{{"base":"/path","adapter":"/path"}}, ...]\'' ) - return [(p["base"], p["adapter"]) for p in entries] + return [ + pytest.param( + ("path", (p["base"], p["adapter"])), id=p["base"].rsplit("/", 1)[-1] + ) + for p in entries + ] BASE_MODEL_PAIRS = _DEFAULT_BASE_MODEL_PAIRS + _load_experimental_pairs() @@ -64,7 +98,6 @@ def _load_experimental_pairs(): @pytest.fixture( scope="module", params=BASE_MODEL_PAIRS, - ids=lambda p: p[0].rsplit("/", 1)[-1], ) def audio_checkpoint(request, tmp_path_factory): """Compose one audio-enabled checkpoint per (base, adapter) pair. @@ -75,8 +108,12 @@ def audio_checkpoint(request, tmp_path_factory): import subprocess import sys - base_model, adapter_library = request.param - save_dir = tmp_path_factory.mktemp(base_model.rsplit("/", 1)[-1]) / "model" + from tests.shared.tiny_asr import build_tiny_asr + + kind, spec = request.param + base_model, adapter_library = spec if kind == "path" else _pair(kind, spec) + save_dir = tmp_path_factory.mktemp(Path(base_model).name) / "model" + tiny_asr = build_tiny_asr(tmp_path_factory.mktemp("tiny-asr") / "asr") cmd = [ sys.executable, @@ -86,7 +123,9 @@ def audio_checkpoint(request, tmp_path_factory): base_model, "--adapters", adapter_library, - "--enable-audio", + "--bundle-asr", + "--asr-model", + str(tiny_asr), "--output", str(save_dir), ] @@ -144,6 +183,13 @@ def _one_completion(outputs): return completion +def test_asr_is_served_from_the_bundle(served): + """The bundled ASR is named in the config the engine serves.""" + config = served["config"] + assert config.asr_bundled_path == "asr" + assert config.asr_model_id + + def test_text_only_serving(served): """Baseline: a plain text request serves normally (backward-compat).""" from vllm import SamplingParams diff --git a/tests/integration/test_audio_uncached_processor.py b/tests/integration/test_audio_uncached_processor.py index 2eba3f41..e78504d3 100644 --- a/tests/integration/test_audio_uncached_processor.py +++ b/tests/integration/test_audio_uncached_processor.py @@ -20,9 +20,15 @@ fails before it can serve anything — booting at all is a large part of the guard. Assertions here are structural (request completes, marker was replaced), never -transcript content: ASR output is not deterministic enough to assert on. +transcript content: ASR output is not deterministic enough to assert on. The +ASR is the tiny local model from ``tests/shared/tiny_asr.py``, bundled into the +checkpoint, so nothing is downloaded for it. -Opt in explicitly: `pytest -m "slow and requires_model and gpu"`. +Two sources (tests/shared/model_sources.py): ``fake`` on every PR run, a tiny +granite-4.0-micro and adapter library built from captured copies of the real +ones; ``real``, the published ones, in the release tier. + +Opt in explicitly: `pytest -m "slow and gpu"`. """ import importlib.util @@ -33,17 +39,17 @@ pytestmark = [ pytest.mark.audio, pytest.mark.slow, - pytest.mark.requires_model, pytest.mark.gpu, ] if importlib.util.find_spec("vllm") is None: pytest.skip("requires vLLM installed", allow_module_level=True) +from tests.shared import model_sources # One small model: this covers a *configuration* dimension, not a model matrix, # and each engine boot here is expensive. -_BASE_MODEL = "ibm-granite/granite-4.0-micro" +_BASE_NAME = "granite-4.0-micro" _ADAPTER_LIBRARY = "ibm-granite/granitelib-core-r1.0" COMPOSE_TIMEOUT_S = 1800 # matches the sibling E2E fixtures @@ -92,22 +98,32 @@ def _cache_disabled_state(llm): return None -@pytest.fixture(scope="module") -def audio_checkpoint(tmp_path_factory): +@pytest.fixture(scope="module", params=model_sources.sources()) +def audio_checkpoint(request, tmp_path_factory): """Compose one audio-enabled checkpoint through the compose CLI.""" import subprocess import sys - save_dir = tmp_path_factory.mktemp(_BASE_MODEL.rsplit("/", 1)[-1]) / "model" + from tests.shared.tiny_asr import build_tiny_asr + + source = request.param + base_model = model_sources.base_model(source, _BASE_NAME) + adapter_library = model_sources.adapter_library( + source, _ADAPTER_LIBRARY, _BASE_NAME + ) + save_dir = tmp_path_factory.mktemp(f"{_BASE_NAME}-{source}") / "model" + tiny_asr = build_tiny_asr(tmp_path_factory.mktemp("tiny-asr") / "asr") cmd = [ sys.executable, "-m", "granite_switch.composer.compose_granite_switch", "--base-model", - _BASE_MODEL, + base_model, "--adapters", - _ADAPTER_LIBRARY, - "--enable-audio", + adapter_library, + "--bundle-asr", + "--asr-model", + str(tiny_asr), "--output", str(save_dir), ] @@ -116,7 +132,7 @@ def audio_checkpoint(tmp_path_factory): ) if result.returncode != 0: raise RuntimeError( - f"compose failed for base={_BASE_MODEL} adapter={_ADAPTER_LIBRARY}\n" + f"compose failed for base={base_model} adapter={adapter_library}\n" f"--- STDOUT ---\n{result.stdout}\n--- STDERR ---\n{result.stderr}" ) return save_dir diff --git a/tests/shared/_vllm_logprobs_worker.py b/tests/shared/_vllm_logprobs_worker.py index 5866e2c8..41206056 100644 --- a/tests/shared/_vllm_logprobs_worker.py +++ b/tests/shared/_vllm_logprobs_worker.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 """Subprocess worker: run ONE vLLM engine and emit teacher-forced prompt logprobs. +One engine may score several prompts: booting it (torch.compile + CUDA-graph +capture) costs far more than scoring, so a caller with several prompts for the +same model passes them all at once. + Invoked as a fresh process per engine so the **pytest process never creates a CUDA context**:: @@ -20,12 +24,19 @@ the pytest process. Contract: -- ``inputs.json``: ``{"input_ids": [int, ...], "vocab_size": int, - "llm_kwargs": {...}}``. ``llm_kwargs`` is forwarded verbatim to ``LLM(...)`` and - carries exactly what the caller passed (e.g. ``max_model_len``, ``enforce_eager``, +- ``inputs.json``: ``{"prompts": [[int, ...], ...], "vocab_size": int, + "llm_kwargs": {...}}``, or the single-prompt form ``{"input_ids": [int, ...], + ...}``. ``llm_kwargs`` is forwarded verbatim to ``LLM(...)`` and carries exactly + what the caller passed (e.g. ``max_model_len``, ``enforce_eager``, ``enable_prefix_caching``, ``gpu_memory_utilization``). - ``--out``: a ``torch.save`` of the dense ``[seq_len-1, vocab_size]`` CPU logprob - tensor, loadable back in the pytest process without CUDA. + tensor, or a list of them (one per prompt, in order) for ``prompts``, loadable + back in the pytest process without CUDA. + +Each prompt gets its own ``generate`` call rather than one batched call. The +equivalence suites gate at ~1e-5, and batching can change padding and kernel +choice enough to move logprobs on its own, so every prompt is scored exactly as +it would be alone; only the engine is shared. """ import argparse @@ -93,7 +104,7 @@ def _pin_single_visible_device(): def cmd_run(args): - """Build one vLLM engine, capture dense prompt logprobs, save to --out.""" + """Build one vLLM engine, capture dense prompt logprobs for each prompt, save.""" # Pin to one GPU BEFORE importing torch/vllm (see _pin_single_visible_device). _pin_single_visible_device() # vLLM v1 forks a subprocess for EngineCore; spawn avoids inheriting any CUDA @@ -112,7 +123,8 @@ def cmd_run(args): with open(args.inputs) as f: spec = json.load(f) - input_ids_list = spec["input_ids"] + batched = "prompts" in spec + prompts = spec["prompts"] if batched else [spec["input_ids"]] vocab_size = spec["vocab_size"] llm_kwargs = dict(spec.get("llm_kwargs") or {}) @@ -142,12 +154,18 @@ def cmd_run(args): temperature=0.0, prompt_logprobs=vocab_size, ) - prompt = TokensPrompt(prompt_token_ids=input_ids_list) - outputs = llm.generate(prompt, sampling_params=sampling_params) - - logprobs_tensor = extract_logprobs_tensor(outputs, vocab_size) - torch.save(logprobs_tensor, args.out) - print(f" saved logprobs {tuple(logprobs_tensor.shape)} to {args.out}") + # One generate per prompt (see the module docstring): the engine is shared, + # the scoring is not. + tensors = [] + for input_ids_list in prompts: + prompt = TokensPrompt(prompt_token_ids=input_ids_list) + outputs = llm.generate(prompt, sampling_params=sampling_params) + tensors.append(extract_logprobs_tensor(outputs, vocab_size)) + torch.save(tensors if batched else tensors[0], args.out) + print( + f" saved logprobs for {len(tensors)} prompt(s) " + f"{[tuple(t.shape) for t in tensors]} to {args.out}" + ) # Synchronous shutdown + device/memory reclaim, in the process that owns the # context. The process then exits, which under Exclusive_Process is what @@ -164,7 +182,9 @@ def main(): p_run = sub.add_parser("run", help="Run one engine, emit dense prompt logprobs") p_run.add_argument("--model", required=True, help="Model dir or HF name to load") p_run.add_argument( - "--inputs", required=True, help="JSON: {input_ids, vocab_size, llm_kwargs}" + "--inputs", + required=True, + help="JSON: {prompts | input_ids, vocab_size, llm_kwargs}", ) p_run.add_argument("--out", required=True, help="Path to torch.save the tensor to") diff --git a/tests/shared/fake_adapter_libraries.py b/tests/shared/fake_adapter_libraries.py new file mode 100644 index 00000000..0860a645 --- /dev/null +++ b/tests/shared/fake_adapter_libraries.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fake granitelib adapter libraries, built from captured copies of the real ones. + +The composer reads an adapter library's *layout and metadata*: the +``///`` tree, each ``adapter_config.json`` (rank, alpha, +target modules, aLoRA invocation tokens) and each ``io.yaml``. The weights it only +stacks. So, like :mod:`tests.shared.fake_bases`, each fake library is the real +repo's small files, captured verbatim into +``tests/shared/fixtures/adapter_libraries//`` by ``capture()``, plus +synthetic weights generated here at test time for a fake base. + +The weights come from :func:`tests.shared.synthetic_adapters.synthesize_adapter` +at the captured rank and target modules, and are fitted to the fake base's own +weight norms. They carry no trained behaviour, which none of the PR-run tests +need; the release tier runs the same tests against the real library. + +Refresh with ``python -m tests.shared.fake_adapter_libraries --capture``. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import shutil +from pathlib import Path + +from tests.shared.base_models import dims_from_config +from tests.shared.synthetic_adapters import ( + PROFILES, + attn_modules, + base_weight_norms, + mlp_modules, + synthesize_adapter, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "adapter_libraries" + +#: Hub repo -> base flavours captured. Only the bases the fake bases cover. +LIBRARIES = { + "ibm-granite/granite-lib-rag-r1.0": ("granite-4.0-micro", "granite-4.1-3b"), + "ibm-granite/granitelib-rag-r1.0": ("granite-4.0-micro", "granite-4.1-3b"), + "ibm-granite/granitelib-core-r1.0": ("granite-4.0-micro", "granite-4.1-3b"), + "ibm-granite/granitelib-guardian-r1.0": ("granite-4.0-micro", "granite-4.1-3b"), +} + +#: Copied verbatim per adapter. The weights are synthesized instead. +CAPTURED_FILES = ("adapter_config.json", "io.yaml") + +#: Library-level files copied when present. None: discovery reads only the +#: per-adapter files, and a captured README links to files not captured. +LIBRARY_FILES: tuple[str, ...] = () + + +def _fixture_dir(repo: str) -> Path: + return FIXTURES / repo.replace("/", "--") + + +def capture(repos=None, revision=None) -> None: + """Refresh the captured files from the Hub. Needs network; not run in tests.""" + from huggingface_hub import hf_hub_download, list_repo_files + + for repo in repos or LIBRARIES: + bases = LIBRARIES[repo] + dest_root = _fixture_dir(repo) + if dest_root.exists(): + shutil.rmtree(dest_root) + files = list_repo_files(repo, revision=revision) + wanted = [ + f + for f in files + if f in LIBRARY_FILES + or ( + f.count("/") == 3 + and f.split("/")[1] in bases + and f.rsplit("/", 1)[1] in CAPTURED_FILES + ) + ] + for f in wanted: + src = hf_hub_download(repo, f, revision=revision) + (dest_root / f).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dest_root / f) + print(f"captured {len(wanted)} files from {repo}") + + +def _profile_for(rank: int, alpha: float): + """A fitted profile at exactly the published ``rank`` and ``alpha``. + + The norm, spectrum and A/B asymmetry come from the fitted profile nearest in + rank; the rank and alpha are the real adapter's, so the synthesized factors + agree with the captured config the composer reads. + """ + nearest = min(PROFILES.values(), key=lambda p: abs(p.rank - rank)) + return dataclasses.replace(nearest, rank=rank, alpha=float(alpha)) + + +def _peft_modules(target_modules: list[str], dims) -> list[str]: + """Map PEFT target names (``q_proj``, ``down_proj`` ...) to synthesizer paths.""" + available = attn_modules() + mlp_modules(dims) + leaf = {m.rsplit(".", 1)[-1]: m for m in available} + return [leaf[t] for t in target_modules if t in leaf] + + +def build_fake_library( + repo: str, + base_name: str, + base_path, + dest, + *, + include: list[str] | None = None, + seed: int = 0, +) -> Path: + """Write a fake ``repo`` library for ``base_name`` to ``dest`` and return it. + + ``base_path`` is the fake base the weights are sized and scaled for (from + :func:`tests.shared.fake_bases.build_fake_base`). ``include`` limits the build + to those adapter names, which keeps a test to the adapters it uses. + """ + from transformers import AutoConfig + + src_root = _fixture_dir(repo) + if not src_root.is_dir(): + raise FileNotFoundError(f"no captured library {repo!r} under {FIXTURES}") + dims = dims_from_config(AutoConfig.from_pretrained(str(base_path))) + norms = base_weight_norms(base_path) + dest = Path(dest) + dest.mkdir(parents=True, exist_ok=True) + for f in LIBRARY_FILES: + if (src_root / f).exists(): + shutil.copyfile(src_root / f, dest / f) + + built = 0 + for config_path in sorted(src_root.glob(f"*/{base_name}/*/adapter_config.json")): + variant_dir = config_path.parent + adapter = variant_dir.parent.parent.name + if include is not None and adapter not in include: + continue + rel = variant_dir.relative_to(src_root) + out = dest / rel + real_config = json.loads(config_path.read_text()) + modules = _peft_modules(real_config["target_modules"], dims) + if not modules: + raise ValueError( + f"{rel}: no target module of {real_config['target_modules']} exists on the base" + ) + profile = _profile_for(real_config["r"], real_config["lora_alpha"]) + synthesize_adapter( + out, + dims, + norms, + profile, + seed=seed + built, + modules=modules, + alora_invocation_tokens=real_config.get("alora_invocation_tokens"), + ) + # The real config, not the synthesizer's: the rank, alpha, target modules + # and invocation tokens the composer reads are the published ones. The + # synthesized factors already have the published rank. + (out / "adapter_config.json").write_text(json.dumps(real_config, indent=2)) + for f in CAPTURED_FILES: + if f != "adapter_config.json" and (variant_dir / f).exists(): + shutil.copyfile(variant_dir / f, out / f) + built += 1 + if built == 0: + raise ValueError( + f"no adapters of {repo!r} for {base_name!r} (include={include})" + ) + return dest + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Refresh the captured adapter-library files." + ) + parser.add_argument("--capture", action="store_true", help="refresh from Hub") + parser.add_argument("--repos", nargs="*", choices=sorted(LIBRARIES)) + args = parser.parse_args() + if args.capture: + capture(args.repos) + + +if __name__ == "__main__": + main() + + +__all__ = ["LIBRARIES", "build_fake_library", "capture"] diff --git a/tests/shared/fake_bases.py b/tests/shared/fake_bases.py new file mode 100644 index 00000000..d44172da --- /dev/null +++ b/tests/shared/fake_bases.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tiny stand-ins for the real Granite bases, built from captured copies of them. + +PR runs must not download real models (see docs/CICD.md, "Release test tier"). The +composer, though, depends on the real bases' *format*: the chat template it injects +control tokens into, the special-token ids it probes, the reserved +``<|unused_N|>`` rows it borrows, the config fields it reads. A tiny random base +with a made-up tokenizer passes none of those checks, so the compose CLI refuses +it, and a test that got past them would prove nothing about the real thing. + +So each fake base is the real repo's small files, captured verbatim into +``tests/shared/fixtures/bases//`` by ``capture()``, plus three things +generated here at test time: + +* **config.json:** the captured config with only the size fields shrunk + (hidden size, layers, heads, intermediate). Every other field, including + ``vocab_size``, ``tie_word_embeddings``, ``torch_dtype`` and the Granite + multipliers, is the real value. +* **tokenizer.json:** a byte-level BPE with no merges, padded so every captured + added token keeps its real id. Text tokenizes to one token per byte, and every + special token, control-token placement and template render behaves as on the + real tokenizer. Only ordinary-word token ids differ. +* **model.safetensors:** seeded random weights of the shrunk shape. + +The release tier runs the same tests on the real bases, which is what catches a +capture that has fallen behind its Hub repo. Refresh with +``python -m tests.shared.fake_bases --capture``. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +from dataclasses import dataclass +from pathlib import Path + +import torch +from safetensors.torch import save_file + +from tests.shared.base_models import _randn + +FIXTURES = Path(__file__).parent / "fixtures" / "bases" + +#: name -> Hub repo the capture came from. The name is what tests pass around. +BASES = { + "granite-4.0-micro": "ibm-granite/granite-4.0-micro", + "granite-4.1-3b": "ibm-granite/granite-4.1-3b", + "granite-4.2-3b": "ibm-granite/granite-4.2-3b", +} + +#: Copied verbatim from the Hub repo. Everything a compose reads except weights +#: and the 7 MB vocabulary. +CAPTURED_FILES = ( + "config.json", + "generation_config.json", + "tokenizer_config.json", + "special_tokens_map.json", + "chat_template.jinja", +) + +#: The size fields shrunk in the fake config. The geometry the vLLM tiny-model +#: tests already serve (tests/shared/granitemoe_compose.py GPU_GEOMETRY): head_dim +#: 32, the smallest the vLLM attention backends accept. Keeps GQA (more query than +#: KV heads), which the real bases all use. +TINY_DIMS = { + "hidden_size": 128, + "intermediate_size": 256, + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, +} + + +def capture(names=None, revision=None) -> None: + """Refresh the captured files from the Hub. Needs network; not run in tests.""" + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import EntryNotFoundError + + for name in names or BASES: + dest = FIXTURES / name + dest.mkdir(parents=True, exist_ok=True) + for filename in CAPTURED_FILES: + try: + src = hf_hub_download(BASES[name], filename, revision=revision) + except EntryNotFoundError: + (dest / filename).unlink(missing_ok=True) + continue + shutil.copyfile(src, dest / filename) + print(f"captured {name} from {BASES[name]}") + + +def _tiny_config(captured: dict) -> dict: + config = dict(captured) + config.update(TINY_DIMS) + layers = TINY_DIMS["num_hidden_layers"] + config["head_dim"] = TINY_DIMS["hidden_size"] // TINY_DIMS["num_attention_heads"] + if "layer_types" in config: + config["layer_types"] = config["layer_types"][:layers] + if "shared_intermediate_size" in config: + config["shared_intermediate_size"] = TINY_DIMS["intermediate_size"] + if "mamba_d_head" in config: + # Unused on an all-attention base, but transformers validates + # mamba_n_heads * mamba_d_head == mamba_expand * hidden_size. + expand = config.get("mamba_expand", 2) + config["mamba_d_head"] = ( + expand * TINY_DIMS["hidden_size"] // config["mamba_n_heads"] + ) + if config["mamba_d_head"] == 0: + config["mamba_n_heads"] = 1 + config["mamba_d_head"] = expand * TINY_DIMS["hidden_size"] + config["max_position_embeddings"] = 4096 + return config + + +def _added_tokens(tokenizer_config: dict) -> dict[int, dict]: + return {int(i): t for i, t in tokenizer_config["added_tokens_decoder"].items()} + + +#: Ordinary words that are single tokens on the real Granite 4 tokenizers (the same +#: id on 4.0, 4.1 and 4.2), and that the tests rely on being single tokens, e.g. +#: as probe labels, which the composer requires to encode to exactly one token. +#: The fake tokenizer keeps each at its real id. +SINGLE_TOKEN_WORDS = { + "safe": 19193, + "unsafe": 39257, + "yes": 9891, + "no": 2201, + "true": 1904, + "false": 3934, + "good": 19045, + "bad": 14176, + "neutral": 60668, + "high": 12156, + "low": 10516, + "medium": 27178, + "left": 2414, + "right": 1315, + "up": 455, +} + + +def _write_tokenizer(dest: Path, tokenizer_config: dict) -> None: + """Byte-level BPE whose added tokens, and :data:`SINGLE_TOKEN_WORDS`, keep + their real ids. Every other word tokenizes to one token per byte.""" + from tokenizers import AddedToken, Tokenizer, decoders, models, pre_tokenizers + + added = _added_tokens(tokenizer_config) + first_added = min(added) + alphabet = sorted(pre_tokenizers.ByteLevel.alphabet()) + vocab = {ch: i for i, ch in enumerate(alphabet)} + # BPE joins bytes only through merges, so each single-token word gets a + # left-to-right merge chain. Its intermediate prefixes take filler ids, which + # are never otherwise produced; the word itself takes its real id. + words = {word: token_id for word, token_id in SINGLE_TOKEN_WORDS.items()} + reserved = set(words.values()) + merges: list[tuple[str, str]] = [] + pieces: list[str] = [] + # Longest first: BPE applies merges by rank, so a word that contains another + # ("unsafe" holds "safe") must have its chain ranked ahead, or the shorter + # word's merges fire first and split it. + for word in sorted(words, key=len, reverse=True): + for k in range(2, len(word) + 1): + merges.append((word[: k - 1], word[k - 1])) + if k < len(word) and word[:k] not in pieces: + pieces.append(word[:k]) + next_free = iter(i for i in range(len(vocab), first_added) if i not in reserved) + for piece in pieces: + vocab[piece] = next(next_free) + vocab.update(words) + # Fillers occupy every remaining id below the first added token, so the added + # tokens land on the ids the real tokenizer gives them. + used = set(vocab.values()) + for i in range(len(alphabet), first_added): + if i not in used: + vocab[f"<|filler_{i}|>"] = i + tokenizer = Tokenizer(models.BPE(vocab=vocab, merges=list(dict.fromkeys(merges)))) + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + tokenizer.decoder = decoders.ByteLevel() + for token_id in sorted(added): + spec = added[token_id] + token = AddedToken( + spec["content"], + lstrip=spec.get("lstrip", False), + rstrip=spec.get("rstrip", False), + normalized=spec.get("normalized", False), + single_word=spec.get("single_word", False), + special=spec.get("special", True), + ) + got = tokenizer.add_special_tokens([token]) + assert got == 1 and tokenizer.token_to_id(spec["content"]) == token_id, ( + f"added token {spec['content']!r} did not land on id {token_id}" + ) + tokenizer.save(str(dest / "tokenizer.json")) + + +@dataclass(frozen=True) +class WeightPreset: + """Random-weight scales for a fake base. + + ``body``: std of the decoder-layer projections. ``embed``: std of the token + embedding (and the untied head). ``final_norm``: the final RMSNorm's gain, + which sets how peaked the next-token distribution is. + """ + + body: float + embed: float + final_norm: float + + +#: Two presets, because on a 2-layer, hidden-128 model with the REAL Granite +#: multipliers (embedding_multiplier 12, residual_multiplier 0.22, logits_scaling +#: 10) no single set of scales gives both properties a trained 40-layer model has. +#: Measured on the tiny geometry: +#: +#: ``REALISTIC`` (the default): body 0.02, embed 0.02, final norm 1. Activations +#: stay small (hidden-state rms ~0.24), which the tests comparing EXACT +#: scores need: test_probe_chunked_prefill checks probe scores to +/- 0.3 +#: against an fp32 reference, test_tp_integration TP=1 vs TP=2 logprobs. +#: But the layers barely reach the output (zeroing every attention or MLP +#: output changes it by JSD 0.000) and the output is near-uniform. +#: ``LIVE_LAYERS``: body 1.0, embed 1.0, final norm 4. Every sublayer matters +#: (dropping attention or the MLP moves the output by JSD 0.98 / 0.99) and +#: the output is peaked (entropy ~4 nats), which tests that need an adapter +#: to MOVE the output depend on: test_multi_adapter_equivalence's `distinct` +#: check. But intermediate hidden states reach rms ~340, where bf16 rounding +#: (~+/-16 at a score of ~2000) breaks those exact-score tolerances. +#: +#: Between the two, the layers either do not matter or the residual grows: the +#: 12x embedding has to be matched for a layer to count. The real model gets both +#: through depth. So a test asks for the one its assertions need. +REALISTIC = WeightPreset(body=0.02, embed=0.02, final_norm=1.0) +LIVE_LAYERS = WeightPreset(body=1.0, embed=1.0, final_norm=4.0) + + +def _scale(key: str, preset: WeightPreset) -> float: + if key in ("model.embed_tokens.weight", "lm_head.weight"): + return preset.embed + return preset.body + + +def _write_weights(dest: Path, config: dict, seed: int, preset: WeightPreset) -> None: + """Seeded random weights of the shrunk shape, keyed per tensor name.""" + hidden = config["hidden_size"] + inter = config["intermediate_size"] + head_dim = config["head_dim"] + q_width = config["num_attention_heads"] * head_dim + kv_width = config["num_key_value_heads"] * head_dim + shared = config["model_type"] == "granitemoehybrid" + vocab = config["vocab_size"] + + shapes = {"model.embed_tokens.weight": (vocab, hidden)} + for layer in range(config["num_hidden_layers"]): + p = f"model.layers.{layer}" + shapes[f"{p}.self_attn.q_proj.weight"] = (q_width, hidden) + shapes[f"{p}.self_attn.k_proj.weight"] = (kv_width, hidden) + shapes[f"{p}.self_attn.v_proj.weight"] = (kv_width, hidden) + shapes[f"{p}.self_attn.o_proj.weight"] = (hidden, q_width) + if shared: + shapes[f"{p}.shared_mlp.input_linear.weight"] = (2 * inter, hidden) + shapes[f"{p}.shared_mlp.output_linear.weight"] = (hidden, inter) + else: + shapes[f"{p}.mlp.gate_proj.weight"] = (inter, hidden) + shapes[f"{p}.mlp.up_proj.weight"] = (inter, hidden) + shapes[f"{p}.mlp.down_proj.weight"] = (hidden, inter) + if not config.get("tie_word_embeddings", True): + shapes["lm_head.weight"] = (vocab, hidden) + + # The captured dtype (bfloat16 on every Granite 4 base), so compose, save and + # load round the same way they do on the real checkpoint. + dtype = getattr(torch, config.get("torch_dtype") or "float32") + sd = { + key: _randn(key, seed, *shape, scale=_scale(key, preset)).to(dtype) + for key, shape in shapes.items() + } + for layer in range(config["num_hidden_layers"]): + p = f"model.layers.{layer}" + sd[f"{p}.input_layernorm.weight"] = torch.ones(hidden, dtype=dtype) + sd[f"{p}.post_attention_layernorm.weight"] = torch.ones(hidden, dtype=dtype) + sd["model.norm.weight"] = torch.full((hidden,), preset.final_norm, dtype=dtype) + save_file(sd, str(dest / "model.safetensors")) + + +def build_fake_base( + name: str, dest, *, seed: int = 0, preset: WeightPreset = REALISTIC +) -> Path: + """Write a loadable tiny ``name`` base to ``dest`` and return its path. + + ``name`` is a key of :data:`BASES`. Composes with the real CLI and loads with + ``from_pretrained``; see the module docstring for what is real and what is not. + ``preset`` picks the random-weight scales (see :data:`REALISTIC`). + """ + src = FIXTURES / name + if not src.is_dir(): + raise FileNotFoundError(f"no captured base {name!r} under {FIXTURES}") + dest = Path(dest) + dest.mkdir(parents=True, exist_ok=True) + for filename in CAPTURED_FILES: + if (src / filename).exists() and filename != "config.json": + shutil.copyfile(src / filename, dest / filename) + + config = _tiny_config(json.loads((src / "config.json").read_text())) + (dest / "config.json").write_text(json.dumps(config, indent=2)) + tokenizer_config = json.loads((src / "tokenizer_config.json").read_text()) + _write_tokenizer(dest, tokenizer_config) + _write_weights(dest, config, seed, preset) + _write_repo_extras(dest) + return dest + + +def _write_repo_extras(dest: Path) -> None: + """Placeholders for the repo files a compose must NOT carry over. + + Every Granite repo ships them, and the composer's upstream-file copy skips + each by a different rule (dotfile, signature extension, excluded name), so a + fake without them would never exercise those exclusions. + """ + (dest / ".gitattributes").write_text("*.safetensors filter=lfs diff=lfs\n") + (dest / "model.sig").write_bytes(b"fake-signature") + (dest / "README.md").write_text("# Fake base for granite-switch tests\n") + # A real, complete index for the single shard: loaders (vLLM among them) + # trust an index when one exists and load only the files it names, so an + # empty weight_map would leave no weights to load at all. + from safetensors import safe_open + + with safe_open(str(dest / "model.safetensors"), framework="pt") as f: + weight_map = {key: "model.safetensors" for key in f.keys()} + (dest / "model.safetensors.index.json").write_text( + json.dumps({"metadata": {}, "weight_map": weight_map}, indent=2) + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Refresh the captured Granite base files." + ) + parser.add_argument("--capture", action="store_true", help="refresh from Hub") + parser.add_argument("--names", nargs="*", choices=sorted(BASES)) + args = parser.parse_args() + if args.capture: + capture(args.names) + + +if __name__ == "__main__": + main() + + +__all__ = [ + "BASES", + "LIVE_LAYERS", + "REALISTIC", + "WeightPreset", + "build_fake_base", + "capture", +] diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/alora/adapter_config.json new file mode 100644 index 00000000..29d9576c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/alora/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/alora/io.yaml new file mode 100644 index 00000000..a4f2b1f3 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/alora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..c5018416 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..01f6a5cc --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + temperature: 0.0 + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..f3f2bae1 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "down_proj", + "gate_proj", + "o_proj", + "v_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..01f6a5cc --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + temperature: 0.0 + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..b90d3f0e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "down_proj", + "o_proj", + "v_proj", + "gate_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..01f6a5cc --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/answerability/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + temperature: 0.0 + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..343c0c3b --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,34 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "up_proj", + "down_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..0bfe018a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,98 @@ +name: citations +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } +transformations: + # Explode the list of document sentences in each citation + - type: explode + input_path: [] # Zero-length path means match root element + target_field: "c" + # Model may repeat itself; drop the resulting duplicates. + - type: drop_duplicates + input_path: [] # Zero-length path means match root element + target_fields: ["r", "c"] + # Replace sentence number with sentence location and contents. + # Do this first for sentences from the last turn, then for sentences from documents. + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + - type: decode_sentences + source: "documents" + input_path: [~, "c"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + document_id: "citation_doc_id" + begin: "citation_begin" + end: "citation_end" + text: "citation_text" + # Remove fields that we no longer need + - type: project + input_path: [] + retained_fields: + - "response_begin" + - "response_end" + - "response_text" + - "citation_doc_id" + - "citation_begin" + - "citation_end" + - "citation_text" + # Merge adjacent document spans + - type: merge_spans + input_path: [] + group_fields: ["response_begin", "response_end", "response_text", "citation_doc_id"] + begin_field: "citation_begin" + end_field: "citation_end" + text_field: "citation_text" + +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the response, identify the statement IDs from the below + documents that it references. Ensure that your output includes all response + sentence IDs, and for each response sentence ID, provide the list of corresponding + referring document sentence IDs. The output must be a json structure. +parameters: + temperature: 0.0 + max_completion_tokens: 4096 +sentence_boundaries: + # Mapping from string location to sentence delimiter prefix + last_message: "r" # , , etc. + documents: "c" + diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..b49c2f7f --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,43 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "v_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..0bfe018a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/citations/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,98 @@ +name: citations +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } +transformations: + # Explode the list of document sentences in each citation + - type: explode + input_path: [] # Zero-length path means match root element + target_field: "c" + # Model may repeat itself; drop the resulting duplicates. + - type: drop_duplicates + input_path: [] # Zero-length path means match root element + target_fields: ["r", "c"] + # Replace sentence number with sentence location and contents. + # Do this first for sentences from the last turn, then for sentences from documents. + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + - type: decode_sentences + source: "documents" + input_path: [~, "c"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + document_id: "citation_doc_id" + begin: "citation_begin" + end: "citation_end" + text: "citation_text" + # Remove fields that we no longer need + - type: project + input_path: [] + retained_fields: + - "response_begin" + - "response_end" + - "response_text" + - "citation_doc_id" + - "citation_begin" + - "citation_end" + - "citation_text" + # Merge adjacent document spans + - type: merge_spans + input_path: [] + group_fields: ["response_begin", "response_end", "response_text", "citation_doc_id"] + begin_field: "citation_begin" + end_field: "citation_end" + text_field: "citation_text" + +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the response, identify the statement IDs from the below + documents that it references. Ensure that your output includes all response + sentence IDs, and for each response sentence ID, provide the list of corresponding + referring document sentence IDs. The output must be a json structure. +parameters: + temperature: 0.0 + max_completion_tokens: 4096 +sentence_boundaries: + # Mapping from string location to sentence delimiter prefix + last_message: "r" # , , etc. + documents: "c" + diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/alora/adapter_config.json new file mode 100644 index 00000000..73de1b4e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/alora/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/alora/io.yaml new file mode 100644 index 00000000..dc5812ee --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/alora/io.yaml @@ -0,0 +1,25 @@ +name: context_relevance +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "ContextRelevanceOutput", + "type": "object", + "properties": { + "context_relevance": { + "type": "string", + "description": "Context relevancy judgment.", + "enum": ["relevant", "irrelevant", "partially relevant"] + } + }, + "required": ["context_relevance"] + } +# Additional turn of instructions to add to the chat +instruction: | + DOCUMENT: {document_content} +# Data transformations to perform during post-processing +transformations: ~ +parameters: + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..a5647a7d --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..dc5812ee --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/context_relevance/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,25 @@ +name: context_relevance +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "ContextRelevanceOutput", + "type": "object", + "properties": { + "context_relevance": { + "type": "string", + "description": "Context relevancy judgment.", + "enum": ["relevant", "irrelevant", "partially relevant"] + } + }, + "required": ["context_relevance"] + } +# Additional turn of instructions to add to the chat +instruction: | + DOCUMENT: {document_content} +# Data transformations to perform during post-processing +transformations: ~ +parameters: + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..27bb78b0 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.2.dev0@e7355a3b2233820f6f30e558ce133ed22673a087", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "input_linear" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..8ddee271 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,73 @@ +name: hallucination_detection +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "HallucinationOutputEntry": { + "properties": { + "r": { + "minimum": 0, + "title": "Sentence Num", + "type": "integer" + }, + "f": { + "title": "Is Faithful", + "type": "string", + "enum": ["faithful", "partial", "unfaithful", "NA"] + }, + "e": { + "title": "Reasoning", + "type": "string" + } + }, + "required": [ + "r", + "e", + "f" + ], + "title": "HallucinationOutputEntry", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/HallucinationOutputEntry" + }, + "title": "HallucinationOutput", + "type": "array" + } +transformations: + # Replace sentence number with sentence location and contents + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + # Remove fields that we no longer need and rename some of the fields. + - type: project + input_path: [] + retained_fields: + "response_begin": "response_begin" + "response_end": "response_end" + "response_text": "response_text" + "f": "faithfulness" + "e": "explanation" +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the last assistant response, identify the faithfulness + by comparing with the provided documents and generate the faithfulness reasoning + and faithfulness decision. + Ensure that your output includes all response sentence IDs, + and for each response sentence ID, provide the corresponding faithfulness + reasoning and faithfulness decision. + The output must be a json structure. +parameters: + # Current LoRA can be quite verbose in its explanations. + max_completion_tokens: 4096 + temperature: 0.0 +sentence_boundaries: + last_message: "i" diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..adff8a53 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..8ddee271 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,73 @@ +name: hallucination_detection +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "HallucinationOutputEntry": { + "properties": { + "r": { + "minimum": 0, + "title": "Sentence Num", + "type": "integer" + }, + "f": { + "title": "Is Faithful", + "type": "string", + "enum": ["faithful", "partial", "unfaithful", "NA"] + }, + "e": { + "title": "Reasoning", + "type": "string" + } + }, + "required": [ + "r", + "e", + "f" + ], + "title": "HallucinationOutputEntry", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/HallucinationOutputEntry" + }, + "title": "HallucinationOutput", + "type": "array" + } +transformations: + # Replace sentence number with sentence location and contents + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + # Remove fields that we no longer need and rename some of the fields. + - type: project + input_path: [] + retained_fields: + "response_begin": "response_begin" + "response_end": "response_end" + "response_text": "response_text" + "f": "faithfulness" + "e": "explanation" +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the last assistant response, identify the faithfulness + by comparing with the provided documents and generate the faithfulness reasoning + and faithfulness decision. + Ensure that your output includes all response sentence IDs, + and for each response sentence ID, provide the corresponding faithfulness + reasoning and faithfulness decision. + The output must be a json structure. +parameters: + # Current LoRA can be quite verbose in its explanations. + max_completion_tokens: 4096 + temperature: 0.0 +sentence_boundaries: + last_message: "i" diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..93e90115 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,44 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@ec5a1b2ce62b14925239f62582eb3db5d9174af2", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "output_linear", + "input_linear", + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..0f40b81c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_clarification +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "clarification": { + "title": "clarification", + "type": "string" + } + }, + "required": [ + "clarification" + ], + "title": "QueryClarificationOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..f0a1b3f3 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..0f40b81c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,24 @@ +name: query_clarification +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "clarification": { + "title": "clarification", + "type": "string" + } + }, + "required": [ + "clarification" + ], + "title": "QueryClarificationOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..bdf7eada --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "q_proj", + "k_proj", + "gate_proj", + "o_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..0f40b81c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_clarification/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_clarification +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "clarification": { + "title": "clarification", + "type": "string" + } + }, + "required": [ + "clarification" + ], + "title": "QueryClarificationOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/adapter_config.json new file mode 100644 index 00000000..87d7ac71 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/adapter_config.json @@ -0,0 +1,44 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..eecbfad4 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,40 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..8a8fe913 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..815aa77e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "down_proj", + "gate_proj", + "q_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granite-lib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..9fb0bdc0 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,38 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..7a4f637c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,100 @@ +name: context-attribution +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } +transformations: + # Explode lists of context sentence indices + - type: explode + input_path: [] # Zero-length path means match root element + target_field: "c" + # Model may repeat itself; drop the resulting duplicates. + - type: drop_duplicates + input_path: [] # Zero-length path means match root element + target_fields: ["r", "c"] + # Replace sentence number with sentence location and contents. + # Do this first for sentences from the last turn, then for sentences from context. + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + - type: decode_sentences + source: + - "documents" + - "all_but_last_message" + input_path: [~, "c"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + document_id: "attribution_doc_id" + message_index: "attribution_msg_index" + begin: "attribution_begin" + end: "attribution_end" + text: "attribution_text" + # Remove fields that we no longer need + - type: project + input_path: [] + retained_fields: + - "response_begin" + - "response_end" + - "response_text" + - "attribution_doc_id" + - "attribution_msg_index" + - "attribution_begin" + - "attribution_end" + - "attribution_text" + +instruction: > + You provided the last assistant response above based on context, which may include documents and/or previous conversation turns. + Your response is divided into sentences, numbered in the format sentence 0 sentence 1 ... + Sentences in the context are also numbered: sentence 0 sentence 1 ... + For each response sentence, please list the context sentences that were most important for you to generate the response sentence. + Provide your answer in JSON format, as an array of JSON objects, where each object has two members: + "r" with the response sentence number as the value, and "c" with an array of context sentence numbers as the value. + An example of such an array of objects is [{{"r": 0, "c": [3, 1, 4]}}, {{"r": 1, "c": [1, 5]}}]. + List the context sentences in order from most important to least important. + Ensure that you include an object for each response sentence, even if the corresponding array of context sentence numbers is empty. + Answer with only the JSON and do not explain. +parameters: + max_completion_tokens: 4096 + temperature: 0.0 +sentence_boundaries: + # Mapping from string location to sentence delimiter prefix + last_message: "r" # , , etc. + documents: "c" + all_but_last_message: "c" diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..e815e8f7 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,42 @@ +{ + "task_type": null, + "peft_type": "LORA", + "auto_mapping": null, + "peft_version": "0.18.1", + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "revision": null, + "inference_mode": false, + "r": 16, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj" + ], + "exclude_modules": null, + "lora_alpha": 32, + "lora_dropout": 0.0, + "fan_in_fan_out": false, + "bias": "none", + "use_rslora": false, + "modules_to_save": null, + "init_lora_weights": true, + "layers_to_transform": null, + "layers_pattern": null, + "rank_pattern": {}, + "alpha_pattern": {}, + "megatron_config": null, + "megatron_core": "megatron.core", + "trainable_token_indices": null, + "loftq_config": {}, + "eva_config": null, + "corda_config": null, + "use_dora": false, + "alora_invocation_tokens": null, + "use_qalora": false, + "qalora_group_size": 16, + "layer_replication": null, + "lora_bias": false, + "target_parameters": null, + "arrow_config": null, + "ensure_weight_tying": false +} diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..7a4f637c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/context-attribution/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,100 @@ +name: context-attribution +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } +transformations: + # Explode lists of context sentence indices + - type: explode + input_path: [] # Zero-length path means match root element + target_field: "c" + # Model may repeat itself; drop the resulting duplicates. + - type: drop_duplicates + input_path: [] # Zero-length path means match root element + target_fields: ["r", "c"] + # Replace sentence number with sentence location and contents. + # Do this first for sentences from the last turn, then for sentences from context. + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + - type: decode_sentences + source: + - "documents" + - "all_but_last_message" + input_path: [~, "c"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + document_id: "attribution_doc_id" + message_index: "attribution_msg_index" + begin: "attribution_begin" + end: "attribution_end" + text: "attribution_text" + # Remove fields that we no longer need + - type: project + input_path: [] + retained_fields: + - "response_begin" + - "response_end" + - "response_text" + - "attribution_doc_id" + - "attribution_msg_index" + - "attribution_begin" + - "attribution_end" + - "attribution_text" + +instruction: > + You provided the last assistant response above based on context, which may include documents and/or previous conversation turns. + Your response is divided into sentences, numbered in the format sentence 0 sentence 1 ... + Sentences in the context are also numbered: sentence 0 sentence 1 ... + For each response sentence, please list the context sentences that were most important for you to generate the response sentence. + Provide your answer in JSON format, as an array of JSON objects, where each object has two members: + "r" with the response sentence number as the value, and "c" with an array of context sentence numbers as the value. + An example of such an array of objects is [{{"r": 0, "c": [3, 1, 4]}}, {{"r": 1, "c": [1, 5]}}]. + List the context sentences in order from most important to least important. + Ensure that you include an object for each response sentence, even if the corresponding array of context sentence numbers is empty. + Answer with only the JSON and do not explain. +parameters: + max_completion_tokens: 4096 + temperature: 0.0 +sentence_boundaries: + # Mapping from string location to sentence delimiter prefix + last_message: "r" # , , etc. + documents: "c" + all_but_last_message: "c" diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..3919d9d0 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,45 @@ +{ + "task_type": null, + "peft_type": "LORA", + "auto_mapping": null, + "peft_version": "0.18.1", + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "revision": null, + "inference_mode": false, + "r": 64, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "input_linear", + "output_linear" + ], + "exclude_modules": null, + "lora_alpha": 64, + "lora_dropout": 0.0, + "fan_in_fan_out": false, + "bias": "none", + "use_rslora": false, + "modules_to_save": null, + "init_lora_weights": true, + "layers_to_transform": null, + "layers_pattern": null, + "rank_pattern": {}, + "alpha_pattern": {}, + "megatron_config": null, + "megatron_core": "megatron.core", + "trainable_token_indices": null, + "loftq_config": {}, + "eva_config": null, + "corda_config": null, + "use_dora": false, + "alora_invocation_tokens": null, + "use_qalora": false, + "qalora_group_size": 16, + "layer_replication": null, + "lora_bias": false, + "target_parameters": null, + "arrow_config": null, + "ensure_weight_tying": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..ecd6d45f --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..d94299c4 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,49 @@ +{ + "alora_invocation_tokens": [ + 27, + 71226, + 29 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.0", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..3f8e4744 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..5c4d8c23 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,43 @@ +{ + "task_type": null, + "peft_type": "LORA", + "auto_mapping": null, + "peft_version": "0.18.1", + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "revision": null, + "inference_mode": false, + "r": 64, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj" + ], + "exclude_modules": null, + "lora_alpha": 64, + "lora_dropout": 0.0, + "fan_in_fan_out": false, + "bias": "none", + "use_rslora": false, + "modules_to_save": null, + "init_lora_weights": true, + "layers_to_transform": null, + "layers_pattern": null, + "rank_pattern": {}, + "alpha_pattern": {}, + "megatron_config": null, + "megatron_core": "megatron.core", + "trainable_token_indices": null, + "loftq_config": {}, + "eva_config": null, + "corda_config": null, + "use_dora": false, + "alora_invocation_tokens": null, + "use_qalora": false, + "qalora_group_size": 16, + "layer_replication": null, + "lora_bias": false, + "target_parameters": null, + "arrow_config": null, + "ensure_weight_tying": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..ecd6d45f --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/requirement-check/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,34 @@ +name: requirement-check +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "requirement_check" +instruction: |- + : {requirement} + Please verify if the assistant's generation satisfies the user's requirements or not and reply with a binary label accordingly. Respond with a json {{"score": "yes"}} if the constraints are satisfied or respond with {{"score": "no"}} if the constraints are not satisfied. +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..7b70f7ef --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,45 @@ +{ + "task_type": null, + "peft_type": "LORA", + "auto_mapping": null, + "peft_version": "0.18.1", + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "revision": null, + "inference_mode": false, + "r": 32, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "input_linear", + "output_linear" + ], + "exclude_modules": null, + "lora_alpha": 64, + "lora_dropout": 0.0, + "fan_in_fan_out": false, + "bias": "none", + "use_rslora": false, + "modules_to_save": null, + "init_lora_weights": true, + "layers_to_transform": null, + "layers_pattern": null, + "rank_pattern": {}, + "alpha_pattern": {}, + "megatron_config": null, + "megatron_core": "megatron.core", + "trainable_token_indices": null, + "loftq_config": {}, + "eva_config": null, + "corda_config": null, + "use_dora": false, + "alora_invocation_tokens": null, + "use_qalora": false, + "qalora_group_size": 16, + "layer_replication": null, + "lora_bias": false, + "target_parameters": null, + "arrow_config": null, + "ensure_weight_tying": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..b2394b86 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..b73d2f71 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": [ + 27, + 12525, + 18773, + 29 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..b2394b86 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..ce356537 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,43 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..b2394b86 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-core-r1.0/uncertainty/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,42 @@ +name: uncertainty +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + # Each 1-digit output maps to 0.1 * + 0.05 + "0": 0.05 + "1": 0.15 + "2": 0.25 + "3": 0.35 + "4": 0.45 + "5": 0.55 + "6": 0.65 + "7": 0.75 + "8": 0.85 + "9": 0.95 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: project + input_path: [] + retained_fields: + score: "certainty" +instruction: +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..8f3adde5 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "input_linear", + "output_linear", + "o_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..08ac6d85 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..d8495b3f --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": [ + 27, + 27190, + 1122, + 29 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "lora_only", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "invocation_string": "", + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..29a5956e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,28 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..14046031 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "v_proj", + "k_proj", + "q_proj", + "gate_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..08ac6d85 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-correction/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,29 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-correction +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Correction Intrinsic", + "type": "object", + "properties": { + "correction": { + "type": "string" + } + }, + "required": ["correction"] + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return a corrected version of the assistant's message based on the given context; otherwise, return 'none'. +parameters: + # corrected response can several hundred tokens at high temperatures + max_completion_tokens: 4096 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..d0ac2387 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "output_linear", + "q_proj", + "input_linear", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..08e3c4ab --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..491e1a81 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": [ + 27, + 27190, + 1122, + 29 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "lora_only", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "invocation_string": "", + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..08e3c4ab --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..5dde274e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.0", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "gate_proj", + "down_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..08e3c4ab --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/factuality-detection/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,30 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +name: factuality-detection +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Factuality Detection Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |2 + + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: A factually incorrect response occurs when the assistant's message contains one or more factual claims that are unsupported by, inconsistent with, or directly contradicted by the information provided in the documents or context. This includes situations where the assistant: introduces details not grounded in the context, misstates or distorts facts contained within the context, misinterprets the meaning or implications of the context, supplies erroneous or conflicting information relative to the context. Even if only a small portion of the response contains such inaccuracies, the overall message is considered factually incorrect. + + ### Scoring Schema: If the last assistant's text meets the criteria, return 'yes'; otherwise, return 'no'. +parameters: + max_completion_tokens: 20 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..ea05706f --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,43 @@ +{ + "task_type": null, + "peft_type": "LORA", + "auto_mapping": null, + "peft_version": "0.18.1", + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "revision": null, + "inference_mode": false, + "r": 32, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj" + ], + "exclude_modules": null, + "lora_alpha": 64, + "lora_dropout": 0.0, + "fan_in_fan_out": false, + "bias": "none", + "use_rslora": false, + "modules_to_save": null, + "init_lora_weights": true, + "layers_to_transform": null, + "layers_pattern": null, + "rank_pattern": {}, + "alpha_pattern": {}, + "megatron_config": null, + "megatron_core": "megatron.core", + "trainable_token_indices": null, + "loftq_config": {}, + "eva_config": null, + "corda_config": null, + "use_dora": false, + "alora_invocation_tokens": null, + "use_qalora": false, + "qalora_group_size": 16, + "layer_replication": null, + "lora_bias": false, + "target_parameters": null, + "arrow_config": null, + "ensure_weight_tying": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..7a568ac8 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..0213807d --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": [ + 27, + 27190, + 1122, + 29 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "o_proj", + "v_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..7a568ac8 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..0ed953e0 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "task_type": null, + "peft_type": "LORA", + "auto_mapping": null, + "peft_version": "0.18.1", + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "revision": null, + "inference_mode": false, + "r": 32, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "exclude_modules": null, + "lora_alpha": 64, + "lora_dropout": 0.0, + "fan_in_fan_out": false, + "bias": "none", + "use_rslora": false, + "modules_to_save": null, + "init_lora_weights": true, + "layers_to_transform": null, + "layers_pattern": null, + "rank_pattern": {}, + "alpha_pattern": {}, + "megatron_config": null, + "megatron_core": "megatron.core", + "trainable_token_indices": null, + "loftq_config": {}, + "eva_config": null, + "corda_config": null, + "use_dora": false, + "alora_invocation_tokens": null, + "use_qalora": false, + "qalora_group_size": 16, + "layer_replication": null, + "lora_bias": false, + "target_parameters": null, + "arrow_config": null, + "ensure_weight_tying": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..7a568ac8 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/guardian-core/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,37 @@ +name: guardian-core +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["yes", "no"] + } + }, + "required": ["score"], + "additionalProperties": false + } +# Output transformation rules to apply +transformations: + - type: likelihood + categories_to_values: + "yes": 1.0 + "no": 0.0 + input_path: ["score"] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "guardian" +instruction: |- + As a judge agent, your role is to help assess whether the provided text meets the given judging criteria, utilizing all available information, including conversations, documents, and tools. + + ### Criteria: {criteria} + + ### Scoring Schema: {scoring_schema} +parameters: + max_completion_tokens: 15 + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..9ed09ae5 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,37 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..c0ddc056 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "label": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["label"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..35d54095 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": [ + 27, + 27190, + 1122, + 29 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..83280bf9 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..64a5ac78 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,43 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "k_proj", + "o_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..83280bf9 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-guardian-r1.0/policy-guardrails/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,36 @@ +name: policy-guardrails +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "Policy Guardrails Intrinsic", + "type": "object", + "properties": { + "score": { + "type": "string", + "enum": ["Yes", "No","Ambiguous"] + } + }, + "required": ["score"], + "additionalProperties": false + } +transformations: ~ +instruction: |- + You are a compliance agent trying to help determine whether a scenario is compliant with a given policy. + + ### Criteria: Policy: {policy_text} + + ### Scoring Schema: Does the scenario described in the previous message comply with the policy? Compliance is determined based on the following guidelines: + : + - "Yes" if, given the scenario and the policy, it can be decided with certainty that the scenario complies with the policy + - "No" if, given the scenario and the policy, it can be decided with certainty that the scenario does not comply with the policy + - "Ambiguous" if, given the scenario and the policy, it is not possible to decide whether the scenario complies with, or violates, the policy and more information is needed to decide with certainty. + + + Your answer must be either "Yes", "No", or "Ambiguous". You MUST return your answer as a valid JSON object with the key "label". For example, if your answer is "Yes", respond as "{{"label":"Yes"}}". +parameters: + temperature: 0.0 + max_completion_tokens: 20 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/alora/adapter_config.json new file mode 100644 index 00000000..29d9576c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/alora/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/alora/io.yaml new file mode 100644 index 00000000..a4f2b1f3 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/alora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 + temperature: 0.0 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..c5018416 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..01f6a5cc --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + temperature: 0.0 + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..f3f2bae1 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "down_proj", + "gate_proj", + "o_proj", + "v_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..01f6a5cc --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + temperature: 0.0 + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..b90d3f0e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "down_proj", + "o_proj", + "v_proj", + "gate_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..01f6a5cc --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/answerability/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,21 @@ +name: answerability +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability" +instruction: ~ +parameters: + temperature: 0.0 + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..343c0c3b --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,34 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "up_proj", + "down_proj", + "gate_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..0bfe018a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,98 @@ +name: citations +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } +transformations: + # Explode the list of document sentences in each citation + - type: explode + input_path: [] # Zero-length path means match root element + target_field: "c" + # Model may repeat itself; drop the resulting duplicates. + - type: drop_duplicates + input_path: [] # Zero-length path means match root element + target_fields: ["r", "c"] + # Replace sentence number with sentence location and contents. + # Do this first for sentences from the last turn, then for sentences from documents. + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + - type: decode_sentences + source: "documents" + input_path: [~, "c"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + document_id: "citation_doc_id" + begin: "citation_begin" + end: "citation_end" + text: "citation_text" + # Remove fields that we no longer need + - type: project + input_path: [] + retained_fields: + - "response_begin" + - "response_end" + - "response_text" + - "citation_doc_id" + - "citation_begin" + - "citation_end" + - "citation_text" + # Merge adjacent document spans + - type: merge_spans + input_path: [] + group_fields: ["response_begin", "response_end", "response_text", "citation_doc_id"] + begin_field: "citation_begin" + end_field: "citation_end" + text_field: "citation_text" + +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the response, identify the statement IDs from the below + documents that it references. Ensure that your output includes all response + sentence IDs, and for each response sentence ID, provide the list of corresponding + referring document sentence IDs. The output must be a json structure. +parameters: + temperature: 0.0 + max_completion_tokens: 4096 +sentence_boundaries: + # Mapping from string location to sentence delimiter prefix + last_message: "r" # , , etc. + documents: "c" + diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..b49c2f7f --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,43 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "v_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..0bfe018a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/citations/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,98 @@ +name: citations +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "_MODEL_OUTPUT_ENTRY": { + "properties": { + "r": { + "minimum": 0, + "title": "R", + "type": "integer" + }, + "c": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "C", + "type": "array" + } + }, + "required": [ + "r", + "c" + ], + "title": "_MODEL_OUTPUT_ENTRY", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/_MODEL_OUTPUT_ENTRY" + }, + "title": "_MODEL_OUTPUT", + "type": "array" + } +transformations: + # Explode the list of document sentences in each citation + - type: explode + input_path: [] # Zero-length path means match root element + target_field: "c" + # Model may repeat itself; drop the resulting duplicates. + - type: drop_duplicates + input_path: [] # Zero-length path means match root element + target_fields: ["r", "c"] + # Replace sentence number with sentence location and contents. + # Do this first for sentences from the last turn, then for sentences from documents. + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + - type: decode_sentences + source: "documents" + input_path: [~, "c"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + document_id: "citation_doc_id" + begin: "citation_begin" + end: "citation_end" + text: "citation_text" + # Remove fields that we no longer need + - type: project + input_path: [] + retained_fields: + - "response_begin" + - "response_end" + - "response_text" + - "citation_doc_id" + - "citation_begin" + - "citation_end" + - "citation_text" + # Merge adjacent document spans + - type: merge_spans + input_path: [] + group_fields: ["response_begin", "response_end", "response_text", "citation_doc_id"] + begin_field: "citation_begin" + end_field: "citation_end" + text_field: "citation_text" + +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the response, identify the statement IDs from the below + documents that it references. Ensure that your output includes all response + sentence IDs, and for each response sentence ID, provide the list of corresponding + referring document sentence IDs. The output must be a json structure. +parameters: + temperature: 0.0 + max_completion_tokens: 4096 +sentence_boundaries: + # Mapping from string location to sentence delimiter prefix + last_message: "r" # , , etc. + documents: "c" + diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/alora/adapter_config.json new file mode 100644 index 00000000..73de1b4e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/alora/adapter_config.json @@ -0,0 +1,45 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/alora/io.yaml new file mode 100644 index 00000000..dc5812ee --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/alora/io.yaml @@ -0,0 +1,25 @@ +name: context_relevance +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "ContextRelevanceOutput", + "type": "object", + "properties": { + "context_relevance": { + "type": "string", + "description": "Context relevancy judgment.", + "enum": ["relevant", "irrelevant", "partially relevant"] + } + }, + "required": ["context_relevance"] + } +# Additional turn of instructions to add to the chat +instruction: | + DOCUMENT: {document_content} +# Data transformations to perform during post-processing +transformations: ~ +parameters: + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..a5647a7d --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,41 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@UNKNOWN", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..dc5812ee --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/context_relevance/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,25 @@ +name: context_relevance +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "title": "ContextRelevanceOutput", + "type": "object", + "properties": { + "context_relevance": { + "type": "string", + "description": "Context relevancy judgment.", + "enum": ["relevant", "irrelevant", "partially relevant"] + } + }, + "required": ["context_relevance"] + } +# Additional turn of instructions to add to the chat +instruction: | + DOCUMENT: {document_content} +# Data transformations to perform during post-processing +transformations: ~ +parameters: + temperature: 0.0 +sentence_boundaries: ~ diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..27bb78b0 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.2.dev0@e7355a3b2233820f6f30e558ce133ed22673a087", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "input_linear" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..8ddee271 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,73 @@ +name: hallucination_detection +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "HallucinationOutputEntry": { + "properties": { + "r": { + "minimum": 0, + "title": "Sentence Num", + "type": "integer" + }, + "f": { + "title": "Is Faithful", + "type": "string", + "enum": ["faithful", "partial", "unfaithful", "NA"] + }, + "e": { + "title": "Reasoning", + "type": "string" + } + }, + "required": [ + "r", + "e", + "f" + ], + "title": "HallucinationOutputEntry", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/HallucinationOutputEntry" + }, + "title": "HallucinationOutput", + "type": "array" + } +transformations: + # Replace sentence number with sentence location and contents + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + # Remove fields that we no longer need and rename some of the fields. + - type: project + input_path: [] + retained_fields: + "response_begin": "response_begin" + "response_end": "response_end" + "response_text": "response_text" + "f": "faithfulness" + "e": "explanation" +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the last assistant response, identify the faithfulness + by comparing with the provided documents and generate the faithfulness reasoning + and faithfulness decision. + Ensure that your output includes all response sentence IDs, + and for each response sentence ID, provide the corresponding faithfulness + reasoning and faithfulness decision. + The output must be a json structure. +parameters: + # Current LoRA can be quite verbose in its explanations. + max_completion_tokens: 4096 + temperature: 0.0 +sentence_boundaries: + last_message: "i" diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..adff8a53 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..8ddee271 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/hallucination_detection/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,73 @@ +name: hallucination_detection +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "$defs": { + "HallucinationOutputEntry": { + "properties": { + "r": { + "minimum": 0, + "title": "Sentence Num", + "type": "integer" + }, + "f": { + "title": "Is Faithful", + "type": "string", + "enum": ["faithful", "partial", "unfaithful", "NA"] + }, + "e": { + "title": "Reasoning", + "type": "string" + } + }, + "required": [ + "r", + "e", + "f" + ], + "title": "HallucinationOutputEntry", + "type": "object" + } + }, + "items": { + "$ref": "#/$defs/HallucinationOutputEntry" + }, + "title": "HallucinationOutput", + "type": "array" + } +transformations: + # Replace sentence number with sentence location and contents + - type: decode_sentences + source: "last_message" + input_path: [~, "r"] # Null in path means wildcard + # New fields to add for each sentence + output_names: + begin: "response_begin" + end: "response_end" + text: "response_text" + # Remove fields that we no longer need and rename some of the fields. + - type: project + input_path: [] + retained_fields: + "response_begin": "response_begin" + "response_end": "response_end" + "response_text": "response_text" + "f": "faithfulness" + "e": "explanation" +instruction: > + Split the last assistant response into individual sentences. + For each sentence in the last assistant response, identify the faithfulness + by comparing with the provided documents and generate the faithfulness reasoning + and faithfulness decision. + Ensure that your output includes all response sentence IDs, + and for each response sentence ID, provide the corresponding faithfulness + reasoning and faithfulness decision. + The output must be a json structure. +parameters: + # Current LoRA can be quite verbose in its explanations. + max_completion_tokens: 4096 + temperature: 0.0 +sentence_boundaries: + last_message: "i" diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..93e90115 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,44 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.17.2.dev0@ec5a1b2ce62b14925239f62582eb3db5d9174af2", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "output_linear", + "input_linear", + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..0f40b81c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_clarification +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "clarification": { + "title": "clarification", + "type": "string" + } + }, + "required": [ + "clarification" + ], + "title": "QueryClarificationOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..f0a1b3f3 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj", + "gate_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..0f40b81c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,24 @@ +name: query_clarification +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "clarification": { + "title": "clarification", + "type": "string" + } + }, + "required": [ + "clarification" + ], + "title": "QueryClarificationOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..bdf7eada --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "q_proj", + "k_proj", + "gate_proj", + "o_proj", + "up_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..0f40b81c --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_clarification/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_clarification +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "clarification": { + "title": "clarification", + "type": "string" + } + }, + "required": [ + "clarification" + ], + "title": "QueryClarificationOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/adapter_config.json new file mode 100644 index 00000000..87d7ac71 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/adapter_config.json @@ -0,0 +1,44 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/alora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/adapter_config.json new file mode 100644 index 00000000..eecbfad4 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/adapter_config.json @@ -0,0 +1,40 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.0-micro", + "bias": "none", + "corda_config": null, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.0-micro/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/adapter_config.json new file mode 100644 index 00000000..8a8fe913 --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": [ + 100264, + 78191, + 100265 + ], + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "down_proj", + "up_proj", + "q_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/alora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/adapter_config.json b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/adapter_config.json new file mode 100644 index 00000000..815aa77e --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "ibm-granite/granite-4.1-3b", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "down_proj", + "gate_proj", + "q_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/io.yaml b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/io.yaml new file mode 100644 index 00000000..098e4b6a --- /dev/null +++ b/tests/shared/fixtures/adapter_libraries/ibm-granite--granitelib-rag-r1.0/query_rewrite/granite-4.1-3b/lora/io.yaml @@ -0,0 +1,24 @@ +name: query_rewrite +# Model name string, or null to use whatever is provided in the chat completion request +model: ~ +# JSON schema of the model's output +response_format: | + { + "properties": { + "rewritten_question": { + "title": "Rewritten Question", + "type": "string" + } + }, + "required": [ + "rewritten_question" + ], + "title": "QueryRewriteOutput", + "type": "object" + } +transformations: ~ +instruction: ~ +parameters: + max_completion_tokens: 1024 + temperature: 0.0 +sentence_boundaries: false diff --git a/tests/shared/fixtures/bases/granite-4.0-micro/chat_template.jinja b/tests/shared/fixtures/bases/granite-4.0-micro/chat_template.jinja new file mode 100644 index 00000000..82e3ebf5 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.0-micro/chat_template.jinja @@ -0,0 +1,118 @@ +{%- set tools_system_message_prefix = 'You are a helpful assistant with access to the following tools. You may call one or more tools to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n' %} +{%- set tools_system_message_suffix = '\n\n\nFor each tool call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request.' %} +{%- set documents_system_message_prefix = 'You are a helpful assistant with access to the following documents. You may use one or more documents to assist with the user query.\n\nYou are given a list of documents within XML tags:\n' %} +{%- set documents_system_message_suffix = '\n\n\nWrite the response to the user\'s input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data.' %} +{%- set g4_default_system_message = 'You are a helpful assistant. Please ensure responses are professional, accurate, and safe.' %} +{%- if available_tools is defined and available_tools %} + {%- set tools = available_tools %} +{%- endif %} +{%- set ns = namespace(tools_system_message=tools_system_message_prefix, + documents_system_message=documents_system_message_prefix, + default_system_message=g4_default_system_message, + system_message='' + ) %} +{%- if tools %} + {%- for tool in tools %} + {%- set ns.tools_system_message = ns.tools_system_message + '\n' + (tool | tojson) %} + {%- endfor %} + {%- set ns.tools_system_message = ns.tools_system_message + tools_system_message_suffix %} +{%- else %} + {%- set ns.tools_system_message = '' %} +{%- endif %} +{%- if documents %} + {%- for document in documents %} + {%- set ns.documents_system_message = ns.documents_system_message + '\n' + (document | tojson) %} + {%- endfor %} + {%- set ns.documents_system_message = ns.documents_system_message + documents_system_message_suffix %} +{%- else %} + {%- set ns.documents_system_message = '' %} +{%- endif %} +{%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {%- set ns.system_message = messages[0].content %} + {%- elif messages[0].content is iterable %} + {%- for entry in messages[0].content %} + {%- if entry.type== 'text' %} + {%- if ns.system_message != '' %} + {%- set ns.system_message = ns.system_message + '\n' %} + {%- endif %} + {%- set ns.system_message = ns.system_message + entry.text %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if tools and documents %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message + '\n\n' + ns.documents_system_message %} + {%- elif tools %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message %} + {%- elif documents %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.documents_system_message %} + {%- endif %} +{%- else %} + {%- if tools and documents %} + {%- set ns.system_message = ns.tools_system_message + '\n\n' + ns.documents_system_message %} + {%- elif tools %} + {%- set ns.system_message = ns.tools_system_message %} + {%- elif documents %} + {%- set ns.system_message = ns.documents_system_message %} + {%- endif %} +{%- endif %} +{%- if ns.system_message %} + {{- '<|start_of_role|>system<|end_of_role|>' + ns.system_message + '<|end_of_text|>\n' }} +{%- else %} + {{- '<|start_of_role|>system<|end_of_role|>' + ns.default_system_message + '<|end_of_text|>\n' }} +{%- endif %} +{%- for message in messages %} + {%- set content = namespace(val='') %} + {%- if message.content is string %} + {%- set content.val = message.content %} + {%- else %} + {%- if message.content is iterable %} + {%- for entry in message.content %} + {%- if entry.type== 'text' %} + {%- if content.val != '' %} + {%- set content.val = content.val + '\n' %} + {%- endif %} + {%- set content.val = content.val + entry.text %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) %} + {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val + '<|end_of_text|>\n' }} + {%- elif message.role == 'assistant' %} + {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content.val) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|end_of_text|>\n' }} + {%- elif message.role == 'tool' %} + {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %} + {{- '<|start_of_role|>user<|end_of_role|>' }} + {%- endif %} + {{- '\n\n' }} + {{- content.val }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %} + {{- '<|end_of_text|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_of_role|>assistant<|end_of_role|>' }} +{%- endif %} \ No newline at end of file diff --git a/tests/shared/fixtures/bases/granite-4.0-micro/config.json b/tests/shared/fixtures/bases/granite-4.0-micro/config.json new file mode 100644 index 00000000..0f2b4321 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.0-micro/config.json @@ -0,0 +1,89 @@ +{ + "architectures": [ + "GraniteMoeHybridForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attention_multiplier": 0.015625, + "bos_token_id": 100257, + "embedding_multiplier": 12, + "eos_token_id": 100257, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.1, + "intermediate_size": 8192, + "layer_types": [ + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention", + "attention" + ], + "logits_scaling": 10, + "mamba_chunk_size": 256, + "mamba_conv_bias": true, + "mamba_d_conv": 4, + "mamba_d_head": 40, + "mamba_d_state": 256, + "mamba_expand": 2, + "mamba_n_groups": 1, + "mamba_n_heads": 128, + "mamba_proj_bias": false, + "max_position_embeddings": 131072, + "model_type": "granitemoehybrid", + "normalization_function": "rmsnorm", + "num_attention_heads": 40, + "num_experts_per_tok": 0, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "num_local_experts": 0, + "output_router_logits": false, + "pad_token_id": 100256, + "position_embedding_type": "rope", + "residual_multiplier": 0.22, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.01, + "shared_intermediate_size": 8192, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.56.0", + "use_cache": true, + "vocab_size": 100352 +} diff --git a/tests/shared/fixtures/bases/granite-4.0-micro/generation_config.json b/tests/shared/fixtures/bases/granite-4.0-micro/generation_config.json new file mode 100644 index 00000000..2eed7ca2 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.0-micro/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 100257, + "eos_token_id": 100257, + "pad_token_id": 100256, + "transformers_version": "4.56.0" +} diff --git a/tests/shared/fixtures/bases/granite-4.0-micro/special_tokens_map.json b/tests/shared/fixtures/bases/granite-4.0-micro/special_tokens_map.json new file mode 100644 index 00000000..3f67e7c5 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.0-micro/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "<|unk|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tests/shared/fixtures/bases/granite-4.0-micro/tokenizer_config.json b/tests/shared/fixtures/bases/granite-4.0-micro/tokenizer_config.json new file mode 100644 index 00000000..7a6b3827 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.0-micro/tokenizer_config.json @@ -0,0 +1,783 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "100256": { + "content": "<|pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100257": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100258": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100259": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100260": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100261": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100262": { + "content": "<|filename|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100263": { + "content": "<|reponame|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100264": { + "content": "<|start_of_role|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100265": { + "content": "<|end_of_role|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100266": { + "content": "<|unused_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100267": { + "content": "<|start_of_plugin|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100268": { + "content": "<|end_of_plugin|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100269": { + "content": "<|unk|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100284": { + "content": "<|unused_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100285": { + "content": "<|unused_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100286": { + "content": "<|unused_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100287": { + "content": "<|unused_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100288": { + "content": "<|unused_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100289": { + "content": "<|unused_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100290": { + "content": "<|unused_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100291": { + "content": "<|unused_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100292": { + "content": "<|unused_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100293": { + "content": "<|unused_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100294": { + "content": "<|unused_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100295": { + "content": "<|unused_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100296": { + "content": "<|unused_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100297": { + "content": "<|unused_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100298": { + "content": "<|unused_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100299": { + "content": "<|unused_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100300": { + "content": "<|unused_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100301": { + "content": "<|unused_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100302": { + "content": "<|unused_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100303": { + "content": "<|unused_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100304": { + "content": "<|unused_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100305": { + "content": "<|unused_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100306": { + "content": "<|unused_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100307": { + "content": "<|unused_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100308": { + "content": "<|unused_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100309": { + "content": "<|unused_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100310": { + "content": "<|unused_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100311": { + "content": "<|unused_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100312": { + "content": "<|unused_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100313": { + "content": "<|unused_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100314": { + "content": "<|unused_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100315": { + "content": "<|unused_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100316": { + "content": "<|unused_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100317": { + "content": "<|unused_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100318": { + "content": "<|unused_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100319": { + "content": "<|unused_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100320": { + "content": "<|unused_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100321": { + "content": "<|unused_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100322": { + "content": "<|unused_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100323": { + "content": "<|unused_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100324": { + "content": "<|unused_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100325": { + "content": "<|unused_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100326": { + "content": "<|unused_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100327": { + "content": "<|unused_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100328": { + "content": "<|unused_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100329": { + "content": "<|unused_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100330": { + "content": "<|unused_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100331": { + "content": "<|unused_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100332": { + "content": "<|unused_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100333": { + "content": "<|unused_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100334": { + "content": "<|unused_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100335": { + "content": "<|unused_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100336": { + "content": "<|unused_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100337": { + "content": "<|unused_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100338": { + "content": "<|unused_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100339": { + "content": "<|unused_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100340": { + "content": "<|unused_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100341": { + "content": "<|unused_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100342": { + "content": "<|unused_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100343": { + "content": "<|unused_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100344": { + "content": "<|unused_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100345": { + "content": "<|unused_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100346": { + "content": "<|unused_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100347": { + "content": "<|unused_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100348": { + "content": "<|unused_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100349": { + "content": "<|unused_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100350": { + "content": "<|unused_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100351": { + "content": "<|unused_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|end_of_text|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "<|pad|>", + "padding_side": "left", + "tokenizer_class": "GPT2Tokenizer", + "unk_token": "<|unk|>" +} diff --git a/tests/shared/fixtures/bases/granite-4.1-3b/chat_template.jinja b/tests/shared/fixtures/bases/granite-4.1-3b/chat_template.jinja new file mode 100644 index 00000000..903cac64 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.1-3b/chat_template.jinja @@ -0,0 +1,114 @@ +{%- set tools_system_message_prefix = 'You are a helpful assistant with access to the following tools. You may call one or more tools to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n' %} +{%- set tools_system_message_suffix = '\n\n\nFor each tool call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request.' %} +{%- set documents_system_message_prefix = 'You are a helpful assistant with access to the following documents. You may use one or more documents to assist with the user query.\n\nYou are given a list of documents within XML tags:\n' %} +{%- set documents_system_message_suffix = '\n\n\nWrite the response to the user\'s input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data.' %} +{%- if available_tools is defined and available_tools %} + {%- set tools = available_tools %} +{%- endif %} +{%- set ns = namespace(tools_system_message=tools_system_message_prefix, + documents_system_message=documents_system_message_prefix, + system_message='' + ) %} +{%- if tools %} + {%- for tool in tools %} + {%- set ns.tools_system_message = ns.tools_system_message + '\n' + (tool | tojson) %} + {%- endfor %} + {%- set ns.tools_system_message = ns.tools_system_message + tools_system_message_suffix %} +{%- else %} + {%- set ns.tools_system_message = '' %} +{%- endif %} +{%- if documents %} + {%- for document in documents %} + {%- set ns.documents_system_message = ns.documents_system_message + '\n' + (document | tojson) %} + {%- endfor %} + {%- set ns.documents_system_message = ns.documents_system_message + documents_system_message_suffix %} +{%- else %} + {%- set ns.documents_system_message = '' %} +{%- endif %} +{%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {%- set ns.system_message = messages[0].content %} + {%- elif messages[0].content is iterable %} + {%- for entry in messages[0].content %} + {%- if entry.type== 'text' %} + {%- if ns.system_message != '' %} + {%- set ns.system_message = ns.system_message + '\n' %} + {%- endif %} + {%- set ns.system_message = ns.system_message + entry.text %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if tools and documents %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message + '\n\n' + ns.documents_system_message %} + {%- elif tools %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message %} + {%- elif documents %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.documents_system_message %} + {%- endif %} +{%- else %} + {%- if tools and documents %} + {%- set ns.system_message = ns.tools_system_message + '\n\n' + ns.documents_system_message %} + {%- elif tools %} + {%- set ns.system_message = ns.tools_system_message %} + {%- elif documents %} + {%- set ns.system_message = ns.documents_system_message %} + {%- endif %} +{%- endif %} +{%- if ns.system_message %} + {{- '<|start_of_role|>system<|end_of_role|>' + ns.system_message + '<|end_of_text|>\n' }} +{%- endif %} +{%- for message in messages %} + {%- set content = namespace(val='') %} + {%- if message.content is string %} + {%- set content.val = message.content %} + {%- else %} + {%- if message.content is iterable %} + {%- for entry in message.content %} + {%- if entry.type== 'text' %} + {%- if content.val != '' %} + {%- set content.val = content.val + '\n' %} + {%- endif %} + {%- set content.val = content.val + entry.text %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) %} + {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val + '<|end_of_text|>\n' }} + {%- elif message.role == 'assistant' %} + {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content.val) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|end_of_text|>\n' }} + {%- elif message.role == 'tool' %} + {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %} + {{- '<|start_of_role|>user<|end_of_role|>' }} + {%- endif %} + {{- '\n\n' }} + {{- content.val }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %} + {{- '<|end_of_text|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_of_role|>assistant<|end_of_role|>' }} +{%- endif %} \ No newline at end of file diff --git a/tests/shared/fixtures/bases/granite-4.1-3b/config.json b/tests/shared/fixtures/bases/granite-4.1-3b/config.json new file mode 100644 index 00000000..89b7e1c3 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.1-3b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "GraniteForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attention_multiplier": 0.015625, + "bos_token_id": 100257, + "embedding_multiplier": 12.0, + "eos_token_id": 100257, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.1, + "intermediate_size": 8192, + "logits_scaling": 10.0, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "granite", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 100256, + "residual_multiplier": 0.22, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000000, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.53.3", + "use_cache": true, + "vocab_size": 100352 +} diff --git a/tests/shared/fixtures/bases/granite-4.1-3b/generation_config.json b/tests/shared/fixtures/bases/granite-4.1-3b/generation_config.json new file mode 100644 index 00000000..dbfc5e2d --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.1-3b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 100257, + "eos_token_id": 100257, + "pad_token_id": 100256, + "transformers_version": "4.53.3" +} diff --git a/tests/shared/fixtures/bases/granite-4.1-3b/special_tokens_map.json b/tests/shared/fixtures/bases/granite-4.1-3b/special_tokens_map.json new file mode 100644 index 00000000..3f67e7c5 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.1-3b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "<|unk|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tests/shared/fixtures/bases/granite-4.1-3b/tokenizer_config.json b/tests/shared/fixtures/bases/granite-4.1-3b/tokenizer_config.json new file mode 100644 index 00000000..7a6b3827 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.1-3b/tokenizer_config.json @@ -0,0 +1,783 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "100256": { + "content": "<|pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100257": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100258": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100259": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100260": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100261": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100262": { + "content": "<|filename|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100263": { + "content": "<|reponame|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100264": { + "content": "<|start_of_role|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100265": { + "content": "<|end_of_role|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100266": { + "content": "<|unused_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100267": { + "content": "<|start_of_plugin|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100268": { + "content": "<|end_of_plugin|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100269": { + "content": "<|unk|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100276": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100277": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100278": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100281": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100282": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100284": { + "content": "<|unused_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100285": { + "content": "<|unused_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100286": { + "content": "<|unused_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100287": { + "content": "<|unused_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100288": { + "content": "<|unused_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100289": { + "content": "<|unused_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100290": { + "content": "<|unused_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100291": { + "content": "<|unused_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100292": { + "content": "<|unused_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100293": { + "content": "<|unused_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100294": { + "content": "<|unused_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100295": { + "content": "<|unused_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100296": { + "content": "<|unused_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100297": { + "content": "<|unused_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100298": { + "content": "<|unused_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100299": { + "content": "<|unused_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100300": { + "content": "<|unused_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100301": { + "content": "<|unused_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100302": { + "content": "<|unused_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100303": { + "content": "<|unused_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100304": { + "content": "<|unused_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100305": { + "content": "<|unused_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100306": { + "content": "<|unused_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100307": { + "content": "<|unused_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100308": { + "content": "<|unused_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100309": { + "content": "<|unused_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100310": { + "content": "<|unused_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100311": { + "content": "<|unused_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100312": { + "content": "<|unused_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100313": { + "content": "<|unused_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100314": { + "content": "<|unused_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100315": { + "content": "<|unused_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100316": { + "content": "<|unused_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100317": { + "content": "<|unused_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100318": { + "content": "<|unused_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100319": { + "content": "<|unused_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100320": { + "content": "<|unused_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100321": { + "content": "<|unused_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100322": { + "content": "<|unused_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100323": { + "content": "<|unused_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100324": { + "content": "<|unused_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100325": { + "content": "<|unused_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100326": { + "content": "<|unused_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100327": { + "content": "<|unused_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100328": { + "content": "<|unused_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100329": { + "content": "<|unused_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100330": { + "content": "<|unused_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100331": { + "content": "<|unused_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100332": { + "content": "<|unused_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100333": { + "content": "<|unused_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100334": { + "content": "<|unused_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100335": { + "content": "<|unused_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100336": { + "content": "<|unused_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100337": { + "content": "<|unused_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100338": { + "content": "<|unused_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100339": { + "content": "<|unused_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100340": { + "content": "<|unused_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100341": { + "content": "<|unused_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100342": { + "content": "<|unused_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100343": { + "content": "<|unused_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100344": { + "content": "<|unused_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100345": { + "content": "<|unused_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100346": { + "content": "<|unused_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100347": { + "content": "<|unused_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100348": { + "content": "<|unused_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100349": { + "content": "<|unused_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100350": { + "content": "<|unused_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100351": { + "content": "<|unused_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|end_of_text|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "<|pad|>", + "padding_side": "left", + "tokenizer_class": "GPT2Tokenizer", + "unk_token": "<|unk|>" +} diff --git a/tests/shared/fixtures/bases/granite-4.2-3b/chat_template.jinja b/tests/shared/fixtures/bases/granite-4.2-3b/chat_template.jinja new file mode 100644 index 00000000..e5c9466d --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.2-3b/chat_template.jinja @@ -0,0 +1,184 @@ +{%- macro tool_to_json(tool) -%} + {%- set ns_tool = namespace(first=true) -%} + {{- '{' -}} + {%- for k, v in tool.items() -%} + {%- if k != 'defer_loading' and k != 'strict' -%} + {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%} + {%- set ns_tool.first = false -%} + {{- '"' ~ k ~ '": ' ~ (v | tojson(ensure_ascii=False)) -}} + {%- endif -%} + {%- endfor -%} + {{- '}' -}} +{%- endmacro -%} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %} +{%- set low_effort = low_effort if low_effort is defined else False %} +{%- if reasoning_effort is defined and reasoning_effort is not none %} + {%- set low_effort = reasoning_effort == "low" %} +{%- endif %} +{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %} + +{%- set ns = namespace(last_user_idx = -1) %} +{%- set loop_messages = messages %} +{%- for m in loop_messages %} + {%- if m["role"] == "user" %} + {%- set ns.last_user_idx = loop.index0 %} + {%- endif %} +{%- endfor %} + +{%- if messages[0]["role"] == "system" %} + {%- set system_message = messages[0]["content"] %} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} + {%- set loop_messages = messages %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = [] %} +{%- endif %} +{# Recompute last_user_idx relative to loop_messages after handling system #} +{%- set ns = namespace(last_user_idx = -1) %} +{%- for m in loop_messages %} + {%- if m["role"] == "user" %} + {%- set ns.last_user_idx = loop.index0 %} + {%- endif %} +{%- endfor %} +{%- if system_message is defined %} + {{- "<|im_start|>system\n" + system_message }} +{%- else %} + {%- if tools is iterable and tools | length > 0 %} + {{- "<|im_start|>system\n" }} + {%- endif %} +{%- endif %} +{%- if tools is iterable and tools | length > 0 %} + {%- if system_message is defined and system_message | length > 0 %} + {{- "\n\n" }} + {%- endif %} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {{- "" }} + {%- for tool in tools %} + {%- if tool.function is defined %} + {%- set tool = tool.function %} + {%- endif %} + {%- if tool.defer_loading is not defined or not tool.defer_loading %} + {{- '\n' ~ tool_to_json(tool) }} + {%- endif %} + {%- endfor %} + {{- "\n" }} + + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} +{%- endif %} + + +{%- if system_message is defined %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if tools is iterable and tools | length > 0 %} + {{- '<|im_end|>\n' }} + {%- endif %} +{%- endif %} + +{%- for message in loop_messages %} + {%- if message.role == "assistant" %} + {# Add reasoning content in to content field for unified processing below. #} + {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %} + {%- set content = "\n" ~ message.reasoning_content ~ "\n\n" ~ (message.content | default('', true)) %} + {%- else %} + {%- set content = message.content | default('', true) %} + {%- if content is string -%} + {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #} + {%- if '' not in content and '' not in content -%} + {%- set content = "" ~ content -%} + {%- endif -%} + {%- else -%} + {%- set content = content -%} + {%- endif -%} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %} + {# Assistant message has tool calls. #} + {{- '<|im_start|>assistant\n' }} + {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %} + {%- if content is string and content | trim | length > 0 %} + {%- if include_content %} + {{- (content | trim) ~ '\n' -}} + {%- else %} + {%- set c = (content | string) %} + {%- if '' in c %} + {# Keep only content after the last closing think. Also generation prompt causes this. #} + {%- set c = c.split('')[-1] %} + {%- elif '' in c %} + {# If was opened but never closed, drop the trailing think segment #} + {%- set c = c.split('')[0] %} + {%- endif %} + {%- set c = "" ~ c | trim %} + {%- if c | length > 0 %} + {{- c ~ '\n' -}} + {%- endif %} + {%- endif %} + {%- else %} + {{- "" -}} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n' -}} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' -}} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value ~ '\n\n' -}} + {%- endfor %} + {%- endif %} + {{- '\n\n' -}} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- else %} + {# Assistant message doesn't have tool calls. #} + {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %} + {{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }} + {%- else %} + {%- set c = (content | default('', true) | string) %} + {%- if '' in c and '' in c %} + {%- set c = "" ~ c.split('')[-1] %} + {%- endif %} + {%- set c = c | trim %} + {%- if c | length > 0 %} + {{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>assistant\n<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- endif %} + {%- elif message.role == "user" or message.role == "system" %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- set content = message.content | string %} + {%- if message.role == "user" and loop.index0 == ns.last_user_idx and low_effort %} + {{- content + '\n\n{reasoning effort: low}' }} + {%- else %} + {{- content }} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user\n' }} + {%- endif %} + {{- '\n' }} + {{- message.content }} + {{- '\n\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }} + {%- endif %} +{%- endfor %} + +{%- if add_generation_prompt %} + {%- if enable_thinking %} + {{- '<|im_start|>assistant\n\n' }} + {%- else %} + {{- '<|im_start|>assistant\n' }} + {%- endif %} +{%- endif %} diff --git a/tests/shared/fixtures/bases/granite-4.2-3b/config.json b/tests/shared/fixtures/bases/granite-4.2-3b/config.json new file mode 100644 index 00000000..ba851b40 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.2-3b/config.json @@ -0,0 +1,37 @@ +{ + "architectures": [ + "GraniteForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attention_multiplier": 0.015625, + "bos_token_id": 100283, + "dtype": "bfloat16", + "embedding_multiplier": 1.0, + "eos_token_id": 100257, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.1, + "intermediate_size": 8192, + "logits_scaling": 1.0, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "granite", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 100257, + "residual_multiplier": 1.0, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "rope_theta": 10000000, + "rope_type": "default" + }, + "rope_scaling": null, + "rope_theta": 10000000, + "tie_word_embeddings": false, + "transformers_version": "4.57.1", + "use_cache": true, + "vocab_size": 100352, + "torch_dtype": "bfloat16" +} diff --git a/tests/shared/fixtures/bases/granite-4.2-3b/generation_config.json b/tests/shared/fixtures/bases/granite-4.2-3b/generation_config.json new file mode 100644 index 00000000..f1d6b1b5 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.2-3b/generation_config.json @@ -0,0 +1,10 @@ +{ + "_from_model_config": true, + "do_sample": true, + "bos_token_id": 100283, + "eos_token_id": 100257, + "pad_token_id": 100257, + "temperature": 1.0, + "top_p": 0.95, + "transformers_version": "4.57.1" +} diff --git a/tests/shared/fixtures/bases/granite-4.2-3b/special_tokens_map.json b/tests/shared/fixtures/bases/granite-4.2-3b/special_tokens_map.json new file mode 100644 index 00000000..0451f379 --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.2-3b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tests/shared/fixtures/bases/granite-4.2-3b/tokenizer_config.json b/tests/shared/fixtures/bases/granite-4.2-3b/tokenizer_config.json new file mode 100644 index 00000000..57e792da --- /dev/null +++ b/tests/shared/fixtures/bases/granite-4.2-3b/tokenizer_config.json @@ -0,0 +1,785 @@ +{ + "add_prefix_space": false, + "added_tokens_decoder": { + "100256": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100257": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100258": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100259": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100260": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100261": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100262": { + "content": "<|filename|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100263": { + "content": "<|reponame|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100264": { + "content": "<|unused_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100265": { + "content": "<|unused_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100266": { + "content": "<|unused_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100267": { + "content": "<|unused_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100268": { + "content": "<|unused_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100269": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100270": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100271": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100272": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100273": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100274": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100275": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100276": { + "content": "[INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100277": { + "content": "[/INST]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100278": { + "content": "[AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100279": { + "content": "[/AVAILABLE_TOOLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100280": { + "content": "[TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100281": { + "content": "[/TOOL_RESULTS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100282": { + "content": "[TOOL_CALLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100283": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100284": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100285": { + "content": "<|unused_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100286": { + "content": "<|unused_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100287": { + "content": "<|unused_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100288": { + "content": "<|unused_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100289": { + "content": "<|unused_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100290": { + "content": "<|unused_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100291": { + "content": "<|unused_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100292": { + "content": "<|unused_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100293": { + "content": "<|unused_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100294": { + "content": "<|unused_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100295": { + "content": "<|unused_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100296": { + "content": "<|unused_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100297": { + "content": "<|unused_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100298": { + "content": "<|unused_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100299": { + "content": "<|unused_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100300": { + "content": "<|unused_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100301": { + "content": "<|unused_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100302": { + "content": "<|unused_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100303": { + "content": "<|unused_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100304": { + "content": "<|unused_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100305": { + "content": "<|unused_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100306": { + "content": "<|unused_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100307": { + "content": "<|unused_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100308": { + "content": "<|unused_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100309": { + "content": "<|unused_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100310": { + "content": "<|unused_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100311": { + "content": "<|unused_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100312": { + "content": "<|unused_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100313": { + "content": "<|unused_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100314": { + "content": "<|unused_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100315": { + "content": "<|unused_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100316": { + "content": "<|unused_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100317": { + "content": "<|unused_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100318": { + "content": "<|unused_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100319": { + "content": "<|unused_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100320": { + "content": "<|unused_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100321": { + "content": "<|unused_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100322": { + "content": "<|unused_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100323": { + "content": "<|unused_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100324": { + "content": "<|unused_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100325": { + "content": "<|unused_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100326": { + "content": "<|unused_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100327": { + "content": "<|unused_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100328": { + "content": "<|unused_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100329": { + "content": "<|unused_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100330": { + "content": "<|unused_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100331": { + "content": "<|unused_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100332": { + "content": "<|unused_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100333": { + "content": "<|unused_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100334": { + "content": "<|unused_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100335": { + "content": "<|unused_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100336": { + "content": "<|unused_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100337": { + "content": "<|unused_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100338": { + "content": "<|unused_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100339": { + "content": "<|unused_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100340": { + "content": "<|unused_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100341": { + "content": "<|unused_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100342": { + "content": "<|unused_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100343": { + "content": "<|unused_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100344": { + "content": "<|unused_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100345": { + "content": "<|unused_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100346": { + "content": "<|unused_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100347": { + "content": "<|unused_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100348": { + "content": "<|unused_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100349": { + "content": "<|unused_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100350": { + "content": "<|unused_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100351": { + "content": "<|unused_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "backend": "tokenizers", + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "is_local": true, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "<|im_end|>", + "padding_side": "left", + "tokenizer_class": "GPT2Tokenizer", + "unk_token": "" +} diff --git a/tests/shared/model_sources.py b/tests/shared/model_sources.py new file mode 100644 index 00000000..094bcc90 --- /dev/null +++ b/tests/shared/model_sources.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +"""One test, two sources: a fake on every PR run, the real model in the release tier. + +PR runs must not download real models; the release tier runs the same tests on the +real ones (docs/CICD.md, "Release test tier"). A test that needs a base model or an +adapter library takes it from here instead of naming a Hub id, and is parametrized +over :data:`SOURCES` with :func:`sources`: + +* ``"fake"``: a tiny base built from the captured real files + (:mod:`tests.shared.fake_bases`) and fake adapter libraries with the captured + real configs (:mod:`tests.shared.fake_adapter_libraries`). Built once per test + session. +* ``"real"``: the Hub id itself, marked ``release`` so PR runs deselect it. + +The two run the same assertions. The fake proves our code against the formats we +captured; the real run proves the captures still match what is published. +""" + +from __future__ import annotations + +import tempfile +from functools import cache +from pathlib import Path + +import pytest + +from tests.shared.fake_adapter_libraries import build_fake_library +from tests.shared.fake_bases import BASES, REALISTIC, WeightPreset, build_fake_base + +SOURCES = ("fake", "real") + +#: A session-wide build directory, shared across tests and xdist-safe per process. +_ROOT = Path(tempfile.mkdtemp(prefix="gs-fake-models-")) + + +def sources(*, real_marks=()): + """``pytest.param`` values for :data:`SOURCES`, with the real one release-marked. + + ``real_marks`` adds marks to the real case only, e.g. ``requires_model``. + """ + return [ + pytest.param("fake", id="fake"), + pytest.param("real", id="real", marks=(pytest.mark.release, *real_marks)), + ] + + +def _preset_tag(preset: WeightPreset) -> str: + return f"b{preset.body:g}-e{preset.embed:g}-n{preset.final_norm:g}" + + +@cache +def _fake_base(name: str, preset: WeightPreset) -> str: + # The directory is named after the model, as a Hub id's last component is: + # the composer derives --target-model from it, which picks the base flavour + # inside an adapter library. So the preset goes in the parent. + dest = _ROOT / "bases" / _preset_tag(preset) / name + return str(build_fake_base(name, dest, preset=preset)) + + +@cache +def _fake_library( + repo: str, base_name: str, include: tuple[str, ...] | None, preset: WeightPreset +) -> str: + tag = "-".join(include) if include else "all" + dest = ( + _ROOT + / "libraries" + / _preset_tag(preset) + / base_name + / repo.replace("/", "--") + / tag + ) + return str( + build_fake_library( + repo, + base_name, + _fake_base(base_name, preset), + dest, + include=list(include) if include else None, + ) + ) + + +def base_model(source: str, name: str, *, preset: WeightPreset = REALISTIC) -> str: + """A path or Hub id for base ``name`` (a key of ``fake_bases.BASES``). + + ``preset`` picks the fake's random-weight scales (``fake_bases.REALISTIC`` or + ``LIVE_LAYERS``); the real source ignores it. + """ + if source == "real": + return BASES[name] + return _fake_base(name, preset) + + +def adapter_library( + source: str, + repo: str, + base_name: str, + *, + include=None, + preset: WeightPreset = REALISTIC, +) -> str: + """A path or Hub id for adapter library ``repo``, built for ``base_name``. + + ``include`` limits a fake library to those adapters; the real library is + passed whole, so a test that needs only some adapters should also pass the + composer's own ``--include-adapters``. + """ + if source == "real": + return repo + return _fake_library(repo, base_name, tuple(include) if include else None, preset) + + +def lora_adapter_dir(source: str, repo: str, base_name: str, adapter: str) -> str: + """The directory of one LoRA adapter of ``repo`` for ``base_name``. + + For tests that need a single adapter directory (a YAML manifest entry, say) + rather than a library. The real source downloads only that adapter. + """ + rel = f"{adapter}/{base_name}/lora" + if source == "real": + from huggingface_hub import snapshot_download + + return str(Path(snapshot_download(repo, allow_patterns=[f"{rel}/*"])) / rel) + return str(Path(adapter_library(source, repo, base_name, include=[adapter])) / rel) + + +__all__ = [ + "SOURCES", + "adapter_library", + "base_model", + "lora_adapter_dir", + "sources", +] diff --git a/tests/shared/tiny_asr.py b/tests/shared/tiny_asr.py new file mode 100644 index 00000000..fc52d3df --- /dev/null +++ b/tests/shared/tiny_asr.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A tiny random CTC ASR model, built locally, for tests that need *an* ASR. + +Stands in for the ~1 GB default where transcript content is not under test: +it loads through the same ``transformers`` ASR pipeline (type ``ctc``), so the +whole cascade runs, but needs no download and is a few hundred KB. Its +transcripts are noise, often empty. +""" + +import json +from pathlib import Path + +_VOCAB = {"": 0, "": 1, "": 2, "": 3, "|": 4, "a": 5, "b": 6} + + +def build_tiny_asr(path) -> Path: + """Write a loadable random Wav2Vec2ForCTC repo to ``path`` and return it.""" + import torch + from transformers import ( + Wav2Vec2Config, + Wav2Vec2CTCTokenizer, + Wav2Vec2FeatureExtractor, + Wav2Vec2ForCTC, + Wav2Vec2Processor, + ) + + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + (path / "vocab.json").write_text(json.dumps(_VOCAB)) + tokenizer = Wav2Vec2CTCTokenizer( + str(path / "vocab.json"), pad_token="", word_delimiter_token="|" + ) + extractor = Wav2Vec2FeatureExtractor( + feature_size=1, sampling_rate=16000, padding_value=0.0, do_normalize=True + ) + Wav2Vec2Processor(feature_extractor=extractor, tokenizer=tokenizer).save_pretrained( + path + ) + config = Wav2Vec2Config( + vocab_size=len(_VOCAB), + pad_token_id=0, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=32, + conv_dim=(16, 16), + conv_stride=(5, 4), + conv_kernel=(10, 4), + num_feat_extract_layers=2, + num_conv_pos_embeddings=16, + num_conv_pos_embedding_groups=2, + ) + torch.manual_seed(0) + Wav2Vec2ForCTC(config).save_pretrained(path, safe_serialization=True) + return path diff --git a/tests/shared/vllm_equivalence.py b/tests/shared/vllm_equivalence.py index f28bb85a..023928bd 100644 --- a/tests/shared/vllm_equivalence.py +++ b/tests/shared/vllm_equivalence.py @@ -269,6 +269,22 @@ def _is_transient_device_error(text: str) -> bool: def run_vllm_logprobs(model_dir, input_ids_list, vocab_size, *, tmpdir, **llm_kwargs): """Run one vLLM engine in a subprocess, return its dense prompt logprobs. + A single-prompt :func:`run_vllm_logprobs_many`; see there. + """ + return run_vllm_logprobs_many( + model_dir, [input_ids_list], vocab_size, tmpdir=tmpdir, **llm_kwargs + )[0] + + +def run_vllm_logprobs_many(model_dir, prompts, vocab_size, *, tmpdir, **llm_kwargs): + """Run ONE vLLM engine in a subprocess, return dense logprobs for each prompt. + + Booting an engine (torch.compile + CUDA-graph capture) costs far more than + scoring a prompt, so a caller with several prompts for the same model should + pass them together. Each prompt is still scored by its own ``generate`` call, + exactly as it would be alone (see _vllm_logprobs_worker.py), so sharing the + engine does not change the numbers the ~1e-5 equivalence gates compare. + The engine is NOT built in this process. Under the GPU host's Exclusive_Process compute mode, if this (the pytest) process ever created a CUDA context -- which ``gpu_mem_util``/``shutdown_llm`` would, via @@ -280,14 +296,15 @@ def run_vllm_logprobs(model_dir, input_ids_list, vocab_size, *, tmpdir, **llm_kw Args: model_dir: path to saved HF model (config.json + model.safetensors) - input_ids_list: list of int token IDs + prompts: list of prompts, each a list of int token IDs vocab_size: vocab size for dense tensor allocation tmpdir: directory for the subprocess handoff files (the caller's tmp_path) **llm_kwargs: extra kwargs for LLM() constructor (e.g. max_model_len, enforce_eager), forwarded verbatim to the worker Returns: - [seq_len-1, vocab_size] float tensor of log-probabilities + list of [seq_len-1, vocab_size] float tensors of log-probabilities, one + per prompt, in order """ # enable_prefix_caching: a SYMMETRIC-config default, applied here (no CUDA) so # both engines of a pair get it. upstream's GraniteMoeHybridForCausalLM declares @@ -318,7 +335,7 @@ def run_vllm_logprobs(model_dir, input_ids_list, vocab_size, *, tmpdir, **llm_kw with open(spec_path, "w") as f: json.dump( { - "input_ids": list(input_ids_list), + "prompts": [list(p) for p in prompts], "vocab_size": vocab_size, "llm_kwargs": llm_kwargs, }, diff --git a/tests/unit/test_asr.py b/tests/unit/test_asr.py index 7d26d750..b0fbdb47 100644 --- a/tests/unit/test_asr.py +++ b/tests/unit/test_asr.py @@ -183,7 +183,7 @@ def test_unrecognized_architecture_becomes_actionable_importerror(self): ) ) with _patched_pipeline(boom): - t = asr.ASRTranscriber(model_id=asr.DEFAULT_ASR_MODEL_ID, device="cpu") + t = asr.ASRTranscriber(model_id="m", device="cpu") with pytest.raises(ImportError) as excinfo: t.load() message = str(excinfo.value) @@ -258,6 +258,113 @@ def test_construction_passes_no_chunk_window(self): assert "chunk_length_s" not in factory.call_args.kwargs +class TestRevisionReachesPipeline: + """The pinned revision is what keeps a checkpoint's transcripts stable when + the ASR repo's main branch moves.""" + + def test_revision_is_passed(self): + factory = mock.Mock(return_value=mock.Mock(type="ctc")) + with _patched_pipeline(factory): + asr.ASRTranscriber(model_id="m", device="cpu", revision="abc").load() + assert factory.call_args.kwargs["revision"] == "abc" + + def test_no_revision_passes_none_through(self): + factory = mock.Mock(return_value=mock.Mock(type="ctc")) + with _patched_pipeline(factory): + asr.ASRTranscriber(model_id="m", device="cpu").load() + assert "revision" not in factory.call_args.kwargs + + def test_pipeline_kwargs_revision_overrides(self): + factory = mock.Mock(return_value=mock.Mock(type="ctc")) + with _patched_pipeline(factory): + asr.ASRTranscriber( + model_id="m", + device="cpu", + revision="abc", + pipeline_kwargs={"revision": "override"}, + ).load() + assert factory.call_args.kwargs["revision"] == "override" + + +class TestResolveAsrSource: + """Where the ASR loads from: bundled weights, else the Hub id, else refuse.""" + + @pytest.fixture(autouse=True) + def _clear_cache(self): + asr.resolve_asr_source.cache_clear() + + def _bundle(self, root): + bundle = root / "asr" + bundle.mkdir() + (bundle / "config.json").write_text("{}") + return root + + def test_hub_id_and_revision(self): + assert asr.resolve_asr_source("org/m", "abc", None, "/ckpt") == ( + "org/m", + "abc", + ) + + def test_missing_model_id_is_refused(self): + with pytest.raises(ValueError) as excinfo: + asr.resolve_asr_source(None, None, None, "/ckpt") + message = str(excinfo.value) + # The error carries the exact fix for a pre-0.2.0 checkpoint. + assert asr.DEFAULT_ASR_MODEL_ID in message + assert asr.DEFAULT_ASR_MODEL_REVISION in message + + def test_bundle_in_local_checkpoint_wins(self, tmp_path): + root = self._bundle(tmp_path) + model, revision = asr.resolve_asr_source("org/m", "abc", "asr", str(root)) + assert model == str(root / "asr") + assert revision is None + + def test_bundle_in_hub_checkpoint_fetches_only_the_subfolder(self, tmp_path): + root = self._bundle(tmp_path) + with mock.patch( + "huggingface_hub.snapshot_download", return_value=str(root) + ) as fetch: + model, _ = asr.resolve_asr_source( + "org/m", "abc", "asr", "org/checkpoint", "rev1" + ) + assert model == str(root / "asr") + fetch.assert_called_once_with( + "org/checkpoint", revision="rev1", allow_patterns=["asr/*"] + ) + + def test_missing_bundle_is_refused(self, tmp_path): + with pytest.raises(FileNotFoundError, match="--bundle-asr"): + asr.resolve_asr_source("org/m", "abc", "asr", str(tmp_path)) + + def test_bundle_without_checkpoint_location_is_refused(self): + with pytest.raises(ValueError, match="checkpoint location"): + asr.resolve_asr_source("org/m", "abc", "asr", None) + + @pytest.mark.parametrize("spelling", ["asr/", "./asr", "asr//", "x/../asr"]) + def test_bundle_path_spellings_resolve_to_the_same_subfolder( + self, tmp_path, spelling + ): + root = self._bundle(tmp_path) + with mock.patch( + "huggingface_hub.snapshot_download", return_value=str(root) + ) as fetch: + model, _ = asr.resolve_asr_source("org/m", None, spelling, "org/ckpt") + assert model == str(root / "asr") + assert fetch.call_args.kwargs["allow_patterns"] == ["asr/*"] + + @pytest.mark.parametrize("bad", ["/asr", "../asr", "asr/../../x", ".", ""]) + def test_bundle_path_outside_the_checkpoint_is_refused(self, tmp_path, bad): + if not bad: + # Empty means "no bundle": it falls through to the Hub id. + assert asr.resolve_asr_source("org/m", None, bad, str(tmp_path)) == ( + "org/m", + None, + ) + return + with pytest.raises(ValueError, match="subdirectory of the checkpoint"): + asr.resolve_asr_source("org/m", None, bad, str(tmp_path)) + + class TestSinglePassCeiling: """The default 120s window is the boundary between 'backend handles it whole' and 'our chunker splits it'.""" @@ -291,9 +398,17 @@ def test_same_key_returns_same_instance(self): b = asr.get_transcriber("m", "cpu") assert a is b - def test_default_model_id_resolution(self): - t = asr.get_transcriber(None, "cpu") - assert t.model_id == asr.DEFAULT_ASR_MODEL_ID + def test_model_id_is_required(self): + with pytest.raises(ValueError, match="model_id"): + asr.get_transcriber(None, "cpu") + + def test_revision_is_part_of_the_key(self): + a = asr.get_transcriber("rev", "cpu", revision="aaa") + b = asr.get_transcriber("rev", "cpu", revision="bbb") + c = asr.get_transcriber("rev", "cpu", revision="aaa") + assert a is not b + assert a is c + assert a.revision == "aaa" def test_different_device_distinct_instance(self): a = asr.get_transcriber("m", "cpu") diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 8637fa21..b8b54c09 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -106,6 +106,8 @@ def test_asr_defaults_off(self): cfg = GraniteSwitchConfig(num_adapters=0) assert cfg.asr_enabled is False assert cfg.asr_model_id is None + assert cfg.asr_model_revision is None + assert cfg.asr_bundled_path is None assert cfg.asr_device == "cuda" assert cfg.asr_dtype is None assert cfg.asr_pipeline_kwargs is None @@ -118,6 +120,20 @@ def test_asr_dtype_round_trip(self, tmp_path): cfg.save_pretrained(tmp_path) assert GraniteSwitchConfig.from_pretrained(tmp_path).asr_dtype == "float32" + def test_asr_source_round_trip(self, tmp_path): + cfg = GraniteSwitchConfig( + num_adapters=0, + asr_enabled=True, + asr_model_id="org/asr", + asr_model_revision="abc123", + asr_bundled_path="asr", + ) + cfg.save_pretrained(tmp_path) + loaded = GraniteSwitchConfig.from_pretrained(tmp_path) + assert loaded.asr_model_id == "org/asr" + assert loaded.asr_model_revision == "abc123" + assert loaded.asr_bundled_path == "asr" + def test_invalid_asr_dtype_raises(self): with pytest.raises(ValueError, match="asr_dtype"): GraniteSwitchConfig(num_adapters=0, asr_dtype="fp8") diff --git a/tests/vllm/_granite4_mini_tests.py b/tests/vllm/_granite4_mini_tests.py index d7f40b7a..5fcfd46d 100644 --- a/tests/vllm/_granite4_mini_tests.py +++ b/tests/vllm/_granite4_mini_tests.py @@ -37,22 +37,97 @@ def _try_import_vllm(): _MODEL_NAMES = sorted(GRANITE4_MINI.keys()) +_SHORT, _LONG = 16, 64 + + +def _random_prompt(seq_len, high): + torch.manual_seed(42) + return torch.randint(0, high, (seq_len,)).tolist() + + +def _adapter_prompt(seq_len): + return make_active_adapter_input(1, seq_len, seed=42)[0].tolist() + + +@pytest.fixture(scope="module", params=_MODEL_NAMES) +def scored(request, tmp_path_factory): + """Every logprob these tests compare, for one variant, from three engines. + + The suite compares three models: the upstream reference ``upstream``, the + adapter-free switch ``switch`` (num_adapters=0), and the zero-LoRA switch + ``zero`` (adapter tiers live, deltas zero). It used to boot a fresh pair of + engines per test -- ten boots per variant, five of them the identical upstream + model -- and an engine boot (torch.compile + CUDA-graph capture) costs far more + than scoring a prompt. So each model is built and booted ONCE here and scores + every prompt its tests need. + + The checkpoints, seeds and prompts are the ones the per-test runners built, + and each prompt still gets its own ``generate`` call (run_vllm_logprobs_many), + so the numbers are the ones the tests compared before. + + The outer test_granite4_mini.py runs this file's classes for a variant in a + single pytest process, so the module scope is what shares the engines. + """ + import gc + + from granite_switch.vllm import register as register_granite_switch + from tests.shared.vllm_equivalence import ( + run_vllm_logprobs_many, + save_switch_model, + save_upstream_model, + save_zero_adapter_model, + ) + + register_granite_switch() + model_name = request.param + cfg = GRANITE4_MINI[model_name] + vocab_size = cfg["vocab_size"] + tmpdir = tmp_path_factory.mktemp(f"granite4_mini_{model_name}") + + upstream_dir, upstream_sd = save_upstream_model(cfg, 0, tmpdir) + switch_dir = save_switch_model(upstream_sd, cfg, tmpdir) + zero_dir = save_zero_adapter_model(upstream_sd, cfg, tmpdir) + del upstream_sd + gc.collect() + + # name -> (prompt, the models that score it) + prompts = { + # TestGranite4FamilyEquivalence: random ids over the full vocabulary + "family_short": (_random_prompt(_SHORT, vocab_size), ("upstream", "switch")), + "family_long": (_random_prompt(_LONG, vocab_size), ("upstream", "switch")), + # TestZeroAdapterNoHiding: random ids clear of the control tokens (101+) + "no_ctrl": (_random_prompt(_SHORT, 100), ("upstream", "zero")), + # TestZeroAdapterEquivalence: control tokens that trigger switching + "ctrl_short": (_adapter_prompt(_SHORT), ("upstream", "zero")), + "ctrl_long": (_adapter_prompt(_LONG), ("upstream", "zero")), + } + dirs = {"upstream": upstream_dir, "switch": switch_dir, "zero": zero_dir} + out = {} + for model, model_dir in dirs.items(): + names = [n for n, (_, models) in prompts.items() if model in models] + tensors = run_vllm_logprobs_many( + model_dir, + [prompts[n][0] for n in names], + vocab_size, + tmpdir=tmpdir, + ) + for name, tensor in zip(names, tensors): + out[(model, name)] = tensor + return {"model_name": model_name, "cfg": cfg, "logprobs": out} -class TestGranite4FamilyEquivalence: - """Integration equivalence: vLLM GraniteSwitch sans LoRA == upstream.""" - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_logits_short(self, model_name, tmp_path): - from tests.shared.vllm_equivalence import run_equivalence_integration +def _pair(scored, prompt, switch_model): + lp = scored["logprobs"] + return lp[("upstream", prompt)], lp[(switch_model, prompt)] - cfg = GRANITE4_MINI[model_name] - layer_types = cfg.get("layer_types", []) - upstream, switch = run_equivalence_integration( - cfg, - seq_len=16, - tmpdir=tmp_path, - ) +class TestGranite4FamilyEquivalence: + """Integration equivalence: vLLM GraniteSwitch sans LoRA == upstream.""" + + def test_logits_short(self, scored): + model_name = scored["model_name"] + layer_types = scored["cfg"].get("layer_types", []) + upstream, switch = _pair(scored, "family_short", "switch") # ULP-equivalent, not bit-exact: the inert switch runs its projections # through the fused SWITCH kernel, whose reduction order differs from @@ -66,18 +141,10 @@ def test_logits_short(self, model_name, tmp_path): msg=f"{model_name}: short sequence logprobs diverge", ) - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_logits_long(self, model_name, tmp_path): - from tests.shared.vllm_equivalence import run_equivalence_integration - - cfg = GRANITE4_MINI[model_name] - layer_types = cfg.get("layer_types", []) - - upstream, switch = run_equivalence_integration( - cfg, - seq_len=64, - tmpdir=tmp_path, - ) + def test_logits_long(self, scored): + model_name = scored["model_name"] + layer_types = scored["cfg"].get("layer_types", []) + upstream, switch = _pair(scored, "family_long", "switch") # ULP-equivalent, not bit-exact (see get_tolerances / test_logits_short). tol = get_tolerances(layer_types, long_sequence=True) @@ -93,18 +160,10 @@ def test_logits_long(self, model_name, tmp_path): class TestZeroAdapterNoHiding: """Zero LoRA weights, adapter infrastructure active.""" - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_no_control_tokens(self, model_name, tmp_path): - from tests.shared.vllm_equivalence import run_zero_adapter_no_hiding_equivalence - - cfg = GRANITE4_MINI[model_name] - layer_types = cfg.get("layer_types", []) - upstream, switch = run_zero_adapter_no_hiding_equivalence( - cfg, - use_control_tokens=False, - seq_len=16, - tmpdir=tmp_path, - ) + def test_no_control_tokens(self, scored): + model_name = scored["model_name"] + layer_types = scored["cfg"].get("layer_types", []) + upstream, switch = _pair(scored, "no_ctrl", "zero") # Skinned + no control token = inert switch. ULP-equivalent to upstream, # not bit-exact: the fused SWITCH kernel reduces in a different order than @@ -123,59 +182,22 @@ def test_no_control_tokens(self, model_name, tmp_path): class TestZeroAdapterEquivalence: """Integration equivalence with active switching and zero LoRA weights.""" - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_logits_short(self, model_name, tmp_path): - from tests.shared.vllm_equivalence import run_zero_adapter_equivalence - - cfg = GRANITE4_MINI[model_name] - layer_types = cfg.get("layer_types", []) - seq_len = 16 - - upstream, switch = run_zero_adapter_equivalence( - cfg, - seq_len=seq_len, - tmpdir=tmp_path, - ) + @pytest.mark.parametrize( + "prompt, seq_len, long_sequence", + [("ctrl_short", _SHORT, False), ("ctrl_long", _LONG, True)], + ids=["short", "long"], + ) + def test_logits(self, scored, prompt, seq_len, long_sequence): + model_name = scored["model_name"] + layer_types = scored["cfg"].get("layer_types", []) + upstream, switch = _pair(scored, prompt, "zero") input_ids = make_active_adapter_input(1, seq_len, seed=42) visible = get_visible_mask(input_ids)[0, :-1] - tol = get_tolerances(layer_types, long_sequence=False, has_kv_hidden=True) - if tol is None: - torch.testing.assert_close( - switch[visible], - upstream[visible], - atol=0.0, - rtol=0.0, - msg=f"{model_name}: logprobs should be bit-exact", - ) - else: - assert_close( - switch[visible], - upstream[visible], - atol=tol[0], - rtol=tol[1], - msg=f"{model_name}: short sequence logprobs diverge (zero-adapter)", - ) - - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_logits_long(self, model_name, tmp_path): - from tests.shared.vllm_equivalence import run_zero_adapter_equivalence - - cfg = GRANITE4_MINI[model_name] - layer_types = cfg.get("layer_types", []) - seq_len = 64 - - upstream, switch = run_zero_adapter_equivalence( - cfg, - seq_len=seq_len, - tmpdir=tmp_path, + tol = get_tolerances( + layer_types, long_sequence=long_sequence, has_kv_hidden=True ) - - input_ids = make_active_adapter_input(1, seq_len, seed=42) - visible = get_visible_mask(input_ids)[0, :-1] - - tol = get_tolerances(layer_types, long_sequence=True, has_kv_hidden=True) if tol is None: torch.testing.assert_close( switch[visible], @@ -190,5 +212,8 @@ def test_logits_long(self, model_name, tmp_path): upstream[visible], atol=tol[0], rtol=tol[1], - msg=f"{model_name}: long sequence logprobs diverge (zero-adapter)", + msg=( + f"{model_name}: {'long' if long_sequence else 'short'} " + "sequence logprobs diverge (zero-adapter)" + ), ) diff --git a/tests/vllm/test_audio_processor.py b/tests/vllm/test_audio_processor.py index ba569b82..cc492215 100644 --- a/tests/vllm/test_audio_processor.py +++ b/tests/vllm/test_audio_processor.py @@ -23,7 +23,6 @@ if _VLLM_AVAILABLE: from granite_switch.vllm.audio import asr as asr_mod from granite_switch.vllm.audio import processor as proc_mod - from granite_switch.vllm.audio.asr import DEFAULT_ASR_MODEL_ID from granite_switch.vllm.audio.processor import ( GraniteSwitchASRMultiModalProcessor, GraniteSwitchASRProcessingInfo, @@ -33,9 +32,12 @@ def _make_info(*, max_model_len=131072, **cfg_attrs): """A ProcessingInfo whose get_hf_config() returns a stub config. - Bypasses __init__, which needs a full vLLM InputProcessingContext. + Bypasses __init__, which needs a full vLLM InputProcessingContext. An ASR + model id is filled in unless given (the composer always writes one); pass + ``asr_model_id=None`` to model a checkpoint without it. """ info = object.__new__(GraniteSwitchASRProcessingInfo) + cfg_attrs.setdefault("asr_model_id", "w") cfg = SimpleNamespace(**cfg_attrs) info.get_hf_config = lambda: cfg info.ctx = SimpleNamespace( @@ -74,13 +76,37 @@ def test_max_tokens_per_item_is_context_derived(self): class TestProcessingInfoAsrAccessors: - def test_model_id_defaults_when_none(self): + def test_missing_model_id_fails_at_startup(self): + # No built-in fallback: get_supported_mm_limits runs at engine startup, + # so a pre-0.2.0 checkpoint is refused before it serves. info = _make_info(asr_enabled=True, asr_model_id=None) - assert info._asr_model_id() == DEFAULT_ASR_MODEL_ID + with pytest.raises(ValueError, match="asr_model_id"): + info.get_supported_mm_limits() - def test_model_id_explicit(self): - info = _make_info(asr_enabled=True, asr_model_id="openai/whisper-small") - assert info._asr_model_id() == "openai/whisper-small" + def test_missing_model_id_is_fine_when_audio_disabled(self): + info = _make_info(asr_enabled=False, asr_model_id=None) + assert info.get_supported_mm_limits() == {} + + def test_source_is_model_id_and_revision(self): + info = _make_info( + asr_enabled=True, + asr_model_id="openai/whisper-small", + asr_model_revision="abc", + ) + assert info._asr_source() == ("openai/whisper-small", "abc") + + def test_source_is_bundle_under_checkpoint(self, tmp_path): + (tmp_path / "asr").mkdir() + (tmp_path / "asr" / "config.json").write_text("{}") + info = _make_info( + asr_enabled=True, + asr_model_id="org/m", + asr_model_revision="abc", + asr_bundled_path="asr", + ) + info.ctx.model_config.model = str(tmp_path) + info.ctx.model_config.revision = None + assert info._asr_source() == (str(tmp_path / "asr"), None) def test_device_default_cuda(self, monkeypatch): # The config default is "cuda". _asr_device() downgrades it to "cpu" when @@ -174,9 +200,10 @@ def transcribe( return "hello world" def fake_get_transcriber( - model_id=None, device="cpu", pipeline_kwargs=None, dtype=None + model_id=None, device="cpu", pipeline_kwargs=None, dtype=None, revision=None ): capture["model_id"] = model_id + capture["revision"] = revision capture["device"] = device capture["pipeline_kwargs"] = pipeline_kwargs capture["dtype"] = dtype @@ -192,6 +219,7 @@ def test_transcribe_forwards_pipeline_and_generate_kwargs(self, monkeypatch): info = _make_info( asr_enabled=True, asr_model_id="whisper-x", + asr_model_revision="rev-x", asr_device="cpu", asr_pipeline_kwargs={"chunk_length_s": 15}, asr_generate_kwargs={}, @@ -201,6 +229,7 @@ def test_transcribe_forwards_pipeline_and_generate_kwargs(self, monkeypatch): ids = proc._transcribe(np.zeros(1600, dtype=np.float32), {"language": "fr"}) assert ids == [1, 2, 3] assert capture["model_id"] == "whisper-x" + assert capture["revision"] == "rev-x" assert capture["pipeline_kwargs"] == {"chunk_length_s": 15} assert capture["generate_kwargs"] == {"language": "fr"} assert capture["sampling_rate"] == proc_mod._TARGET_SR @@ -364,9 +393,7 @@ def transcribe( monkeypatch.setattr( proc_mod, "get_transcriber", - lambda model_id=None, device="cpu", pipeline_kwargs=None, dtype=None: ( - FakeTranscriber() - ), + lambda **_kwargs: FakeTranscriber(), ) return proc @@ -478,9 +505,7 @@ def transcribe(self, audio, **kw): monkeypatch.setattr( proc_mod, "get_transcriber", - lambda model_id=None, device="cpu", pipeline_kwargs=None, dtype=None: ( - FakeTranscriber() - ), + lambda **_kwargs: FakeTranscriber(), ) bf = proc._call_hf_processor( diff --git a/tests/vllm/test_generation_equivalence.py b/tests/vllm/test_generation_equivalence.py index fc93da13..4a2215d5 100644 --- a/tests/vllm/test_generation_equivalence.py +++ b/tests/vllm/test_generation_equivalence.py @@ -17,7 +17,11 @@ Worker: _generation_equivalence_worker.py (build / run / compare phases). -Markers: requires_model (consistent with test_skinning_equivalence.py). +Markers: requires_model (consistent with test_skinning_equivalence.py), and +release: CLAUDE.md calls this the authoritative equivalence test, but it compares +against upstream on the real weights, with thresholds calibrated on them. On +random weights the same equivalence runs on every PR in +tests/vllm/test_granite4_mini.py. """ import os @@ -134,6 +138,7 @@ def _run_generation_test(model_name, timeout): ) +@pytest.mark.release @pytest.mark.requires_model @pytest.mark.parametrize("model_name", MODELS, ids=_short_name) def test_generation_equivalence(model_name): diff --git a/tests/vllm/test_granite4_fullsize.py b/tests/vllm/test_granite4_fullsize.py index c190501d..eb1eef12 100644 --- a/tests/vllm/test_granite4_fullsize.py +++ b/tests/vllm/test_granite4_fullsize.py @@ -90,6 +90,9 @@ def _run_inner_class(class_name): assert result.returncode == 0, f"Inner tests failed (exit {result.returncode})" +# Release tier: the same equivalence runs on every PR at small dimensions +# (test_granite4_mini.py); full size adds only scale, at ~12 min a run. +@pytest.mark.release @pytest.mark.skipif(not _VLLM_AVAILABLE, reason="requires vLLM installed") class TestGranite4FullSizeEquivalence: def test_suite(self): diff --git a/tests/vllm/test_granite4_mini.py b/tests/vllm/test_granite4_mini.py index dac246bd..61bb98f7 100644 --- a/tests/vllm/test_granite4_mini.py +++ b/tests/vllm/test_granite4_mini.py @@ -40,6 +40,15 @@ _MODEL_NAMES = sorted(GRANITE4_MINI.keys()) +# The vLLM equivalence classes cost minutes per variant, so PR runs keep one +# representative and the release tier runs the rest. The CPU weight-transfer +# classes stay on every variant: they take seconds. +_PR_MODEL = "4.0-micro" +_GPU_MODEL_PARAMS = [ + pytest.param(name, marks=() if name == _PR_MODEL else pytest.mark.release) + for name in _MODEL_NAMES +] + # ── Weight transfer tests (HF-level, no vLLM) ──────────────────── @@ -103,13 +112,15 @@ def test_weight_transfer(self, model_name): gc.collect() -# ── GPU test class wrappers (subprocess) ────────────────────────── +# ── GPU tests (subprocess) ──────────────────────────────────────── _INNER = Path(__file__).parent / "_granite4_mini_tests.py" -_TIMEOUT = 1200 +# One subprocess now runs all three classes for a variant (three engine boots in +# all), where each class used to get its own subprocess and its own two engines. +_TIMEOUT = 1800 -def _run_inner_class(class_name): +def _run_inner(selector): cmd = [ sys.executable, "-m", @@ -119,32 +130,27 @@ def _run_inner_class(class_name): "-s", "--tb=short", "-k", - class_name, + selector, ] result = subprocess.run(cmd, capture_output=True, text=True, timeout=_TIMEOUT) if result.stdout: - print(result.stdout[-4000:]) + print(result.stdout[-6000:]) if result.stderr: print("STDERR:", result.stderr[-2000:]) assert result.returncode == 0, f"Inner tests failed (exit {result.returncode})" @pytest.mark.skipif(not _VLLM_AVAILABLE, reason="requires vLLM installed") -class TestGranite4FamilyEquivalence: - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_suite(self, model_name): - _run_inner_class(f"TestGranite4FamilyEquivalence and {model_name}") - - -@pytest.mark.skipif(not _VLLM_AVAILABLE, reason="requires vLLM installed") -class TestZeroAdapterNoHiding: - @pytest.mark.parametrize("model_name", _MODEL_NAMES) - def test_suite(self, model_name): - _run_inner_class(f"TestZeroAdapterNoHiding and {model_name}") - +class TestGranite4VllmEquivalence: + """All vLLM equivalence classes for one variant, in ONE pytest process. + + TestGranite4FamilyEquivalence, TestZeroAdapterNoHiding and + TestZeroAdapterEquivalence in _granite4_mini_tests.py share a module fixture + that boots each of the variant's three models once. Running them in a single + process is what lets them share it: per-class processes would rebuild it, + and the boots, three times. + """ -@pytest.mark.skipif(not _VLLM_AVAILABLE, reason="requires vLLM installed") -class TestZeroAdapterEquivalence: - @pytest.mark.parametrize("model_name", _MODEL_NAMES) + @pytest.mark.parametrize("model_name", _GPU_MODEL_PARAMS) def test_suite(self, model_name): - _run_inner_class(f"TestZeroAdapterEquivalence and {model_name}") + _run_inner(model_name) diff --git a/tests/vllm/test_multi_adapter_equivalence.py b/tests/vllm/test_multi_adapter_equivalence.py index 5d398862..c2e5c7da 100644 --- a/tests/vllm/test_multi_adapter_equivalence.py +++ b/tests/vllm/test_multi_adapter_equivalence.py @@ -93,6 +93,9 @@ import pytest +from tests.shared import model_sources +from tests.shared.fake_bases import LIVE_LAYERS + WORKER = Path(__file__).parent / "_multi_adapter_equivalence_worker.py" #: One base per Granite chat-template family: 4.1-3b renders the #: ``<|start_of_role|>`` role markers, 4.2-3b the ChatML ``<|im_start|>`` form. Both @@ -118,7 +121,51 @@ RUN_TIMEOUT = 2400 # vLLM load + one scored generate per case COMPARE_TIMEOUT = 300 -CASES = [(base, flavor) for base in BASE_MODELS for flavor in FLAVORS] +# PR runs use a fake of the first default base (role-marker template), built +# from captured copies of the real repo (tests/shared/model_sources.py); the +# release tier runs every base for real (ChatML 4.2-3b included). An explicit +# MULTI_ADAPTER_BASE_MODELS runs exactly the bases it names, unmarked. +_EXPLICIT = "MULTI_ADAPTER_BASE_MODELS" in os.environ + + +def _cases(): + if _EXPLICIT: + return [ + pytest.param((base, flavor), id=f"{base.split('/')[-1]}-{flavor}") + for base in BASE_MODELS + for flavor in FLAVORS + ] + pr_name = BASE_MODELS[0].split("/")[-1] + cases = [ + pytest.param(("fake", pr_name, flavor), id=f"{pr_name}-{flavor}-fake") + for flavor in FLAVORS + ] + cases += [ + pytest.param( + ("real", base, flavor), + marks=pytest.mark.release, + id=f"{base.split('/')[-1]}-{flavor}-real", + ) + for base in BASE_MODELS + for flavor in FLAVORS + ] + return cases + + +CASES = _cases() + + +def _resolve_case(case): + """``(base path or repo id, flavor)`` for a case.""" + if len(case) == 2: + return case + source, base, flavor = case + if source == "fake": + # LIVE_LAYERS: the `distinct` check needs every adapter to move the output, + # which the default fake base's barely-reached layers cannot do (see + # tests/shared/fake_bases.py). + return model_sources.base_model("fake", base, preset=LIVE_LAYERS), flavor + return base, flavor def _run_step(step_name, *cmd_args, timeout): @@ -139,9 +186,7 @@ def _run_step(step_name, *cmd_args, timeout): ) -@pytest.fixture( - scope="module", params=CASES, ids=lambda c: f"{c[0].split('/')[-1]}-{c[1]}" -) +@pytest.fixture(scope="module", params=CASES) def flavor_run(request): """Compose both checkpoints and score all three models once per case. @@ -149,7 +194,7 @@ def flavor_run(request): four checks read the same captured distributions. Only one parametrization is live at a time, so at most two composed checkpoints sit on disk at once. """ - base, flavor = request.param + base, flavor = _resolve_case(request.param) label = f"{flavor}@{base}" with tempfile.TemporaryDirectory(prefix="multi_adapter_") as work_dir: _run_step( @@ -239,13 +284,11 @@ def _check(flavor_run, check): ) -@pytest.mark.requires_model def test_no_control_token_equals_upstream(flavor_run): """Adapter id 0 is a true no-op even with several trained adapters embedded.""" _check(flavor_run, "passthrough") -@pytest.mark.requires_model def test_adapters_are_distinguishable(flavor_run): """Every adapter changes the distribution, and no two adapters agree. @@ -254,13 +297,11 @@ def test_adapters_are_distinguishable(flavor_run): _check(flavor_run, "distinct") -@pytest.mark.requires_model def test_output_is_independent_of_adapter_slot(flavor_run): """The same adapter at a different slot produces the same distribution.""" _check(flavor_run, "slots") -@pytest.mark.requires_model def test_output_is_independent_of_batch_composition(flavor_run): """A prompt is unaffected by other adapters sharing its batch. diff --git a/tests/vllm/test_probe_chunked_prefill.py b/tests/vllm/test_probe_chunked_prefill.py index 53a54f96..fa02b16d 100644 --- a/tests/vllm/test_probe_chunked_prefill.py +++ b/tests/vllm/test_probe_chunked_prefill.py @@ -14,6 +14,11 @@ Each budget runs in its own subprocess so only one vLLM engine is resident. +Two sources for the base-model tests (tests/shared/model_sources.py): ``fake`` on +every PR run, a tiny granite-4.1-3b built from captured copies of the real repo; +``real``, the published one, in the release tier. ``CLS_TEST_BASE`` overrides the +real base. The SR tests use their own synthetic base. + Requires GPU + vLLM. """ @@ -31,7 +36,31 @@ WORKER = os.path.join(os.path.dirname(__file__), "_probe_chunked_prefill_worker.py") _REPO = os.path.dirname(os.path.dirname(os.path.dirname(WORKER))) -BASE_MODEL = os.environ.get("CLS_TEST_BASE", "ibm-granite/granite-4.1-3b") +from tests.shared import model_sources + +_BASE_NAME = "granite-4.1-3b" +_REAL_BASE = os.environ.get("CLS_TEST_BASE", model_sources.BASES[_BASE_NAME]) + + +@pytest.fixture(scope="module", params=model_sources.sources()) +def base_model(request): + """The base the base-model probe checkpoints compose on, per source.""" + if request.param == "real": + return _REAL_BASE + return model_sources.base_model("fake", _BASE_NAME) + + +def _mid_layer(base, fraction): + """A decoder layer at ``fraction`` of the base's depth. + + The intermediate-layer tests read a middle layer of the real 40-layer base; + the fake base has only a couple, so the layer scales with the base. + """ + from transformers import AutoConfig + + layers = AutoConfig.from_pretrained(base).num_hidden_layers + return max(0, min(layers - 1, int(layers * fraction))) + # One budget per distinct final-chunk shape: ``None`` disables chunked prefill # (single pass); 256 leaves a wide final chunk; 101 a narrow one; 48 splits into many @@ -47,12 +76,19 @@ def _env(): return env -def _compose(tmp_path_factory, argv, tag): - """Compose a probe checkpoint in a CPU subprocess, before any CUDA init.""" +def _compose(tmp_path_factory, argv, tag, base=None): + """Compose a probe checkpoint in a CPU subprocess, before any CUDA init. + + ``base`` is the base model for the base-model composes; the SR composes build + their own synthetic base and ignore it. + """ outdir = str(tmp_path_factory.mktemp(tag)) + env = {**_env(), "CLS_OUT": outdir} + if base is not None: + env["CLS_BASE"] = base proc = subprocess.run( [sys.executable, WORKER, *argv], - env={**_env(), "CLS_BASE": BASE_MODEL, "CLS_OUT": outdir}, + env=env, capture_output=True, text=True, timeout=1800, @@ -107,9 +143,11 @@ def _run_worker(outdir, argv, env, timeout=1800): @pytest.fixture(scope="module") -def composed(tmp_path_factory): +def composed(base_model, tmp_path_factory): """Multi-label probe over a varied prompt.""" - result = _compose(tmp_path_factory, ["compose-multilabel"], "cls_ckpt") + result = _compose( + tmp_path_factory, ["compose-multilabel"], "cls_ckpt", base=base_model + ) result["pids_path"] = _write_pids(result) return result @@ -447,7 +485,7 @@ def gap(scores): print(f" activation contribution: {vllm_gap - hf_gap:.4f}") -def test_probe_reads_an_intermediate_layer_correctly(tmp_path_factory): +def test_probe_reads_an_intermediate_layer_correctly(base_model, tmp_path_factory): """A layer-20 slot under eager and FULL, against an fp32 HF layer-20 reference. The fixed seed-0 head's reference margin (about 0.07) is below the score @@ -455,9 +493,12 @@ def test_probe_reads_an_intermediate_layer_correctly(tmp_path_factory): layer 20 against nearby layers; it shows the path serves in both modes with a score gap within tolerance of the layer-20 reference. """ - read_layer = 20 # a middle layer of the 40-layer base + read_layer = _mid_layer(base_model, 0.5) # layer 20 of the real 40-layer base result = _compose( - tmp_path_factory, ["compose-at-layer", str(read_layer)], "cls_layer_ckpt" + tmp_path_factory, + ["compose-at-layer", str(read_layer)], + "cls_layer_ckpt", + base=base_model, ) label_token_ids = result["label_token_ids"] env = { @@ -483,7 +524,9 @@ def test_probe_reads_an_intermediate_layer_correctly(tmp_path_factory): assert not failures, "\n".join(failures) -def test_two_probe_slots_replay_correctly_in_one_captured_shape(tmp_path_factory): +def test_two_probe_slots_replay_correctly_in_one_captured_shape( + base_model, tmp_path_factory +): """Serve a final-layer and a layer-15 slot under eager and FULL. Probe arithmetic and transport are checked against the head's @@ -500,9 +543,12 @@ def test_two_probe_slots_replay_correctly_in_one_captured_shape(tmp_path_factory reference. Slot 1's HF/vLLM activation difference is printed, with no acceptance threshold. The test does not prove which captured graph replayed. """ - read_layer = 15 + read_layer = _mid_layer(base_model, 0.375) # layer 15 of the real 40-layer base result = _compose( - tmp_path_factory, ["compose-two-slot", str(read_layer)], "cls_two_slot_ckpt" + tmp_path_factory, + ["compose-two-slot", str(read_layer)], + "cls_two_slot_ckpt", + base=base_model, ) label_token_ids = result["label_token_ids"] env = { diff --git a/tests/vllm/test_tp_integration.py b/tests/vllm/test_tp_integration.py index 4869f26a..f99b82fe 100644 --- a/tests/vllm/test_tp_integration.py +++ b/tests/vllm/test_tp_integration.py @@ -29,6 +29,8 @@ import pytest import torch +from tests.shared import model_sources + _VLLM_AVAILABLE = importlib.util.find_spec("vllm") is not None _NUM_GPUS = torch.cuda.device_count() if torch.cuda.is_available() else 0 @@ -202,17 +204,12 @@ class TestTPRealAdapters: sharding too. TP=1 and TP=2 logprobs must agree. """ - def test_tp_logprobs_agree(self, tmp_path): + @pytest.mark.parametrize("source", model_sources.sources()) + def test_tp_logprobs_agree(self, tmp_path, source): _build_and_compare( str(tmp_path), - build_args=[ - "build-compose", - "--base-model", - "ibm-granite/granite-4.0-micro", - "--adapter-repos", - "ibm-granite/granitelib-rag-r1.0", - ], - label="granite-4.0-micro-rag", + build_args=_rag_build_args(source), + label=f"granite-4.0-micro-rag-{source}", intrinsic_name="answerability", ) @@ -232,21 +229,36 @@ class TestTPMultiSwitch: control-token placement is exercised under sharding too. """ - def test_tp_logprobs_agree_multi(self, tmp_path): + @pytest.mark.parametrize("source", model_sources.sources()) + def test_tp_logprobs_agree_multi(self, tmp_path, source): _build_and_compare( str(tmp_path), - build_args=[ - "build-compose", - "--base-model", - "ibm-granite/granite-4.0-micro", - "--adapter-repos", - "ibm-granite/granitelib-rag-r1.0", - ], - label="granite-4.0-micro-rag-multi", + build_args=_rag_build_args(source), + label=f"granite-4.0-micro-rag-multi-{source}", intrinsic_name="answerability", ) +_BASE_NAME = "granite-4.0-micro" +_RAG_LIBRARY = "ibm-granite/granitelib-rag-r1.0" + + +def _rag_build_args(source): + """Compose args for 4.0-micro + the RAG library, per source. + + ``fake`` (every PR run) is a tiny base and library built from captured copies + of the real ones (tests/shared/model_sources.py); its geometry (4 query, 2 KV + heads) shards evenly at TP=2. ``real`` runs in the release tier. + """ + return [ + "build-compose", + "--base-model", + model_sources.base_model(source, _BASE_NAME), + "--adapter-repos", + model_sources.adapter_library(source, _RAG_LIBRARY, _BASE_NAME), + ] + + class TestTPGraniteMoe: """TP=1 vs TP=2 on a pure sparse MoE base (``granitemoe``), both adaptations. diff --git a/uv.lock b/uv.lock index 8e5f16b9..7c91d152 100644 --- a/uv.lock +++ b/uv.lock @@ -1575,7 +1575,7 @@ wheels = [ [[package]] name = "granite-switch" -version = "0.1.1" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-14-granite-switch-dev' or extra == 'group-14-granite-switch-test' or extra == 'group-14-granite-switch-vllm26' or (extra == 'group-14-granite-switch-dev-latest' and extra == 'group-14-granite-switch-dev-vllm27') or (extra == 'group-14-granite-switch-dev-latest' and extra == 'group-14-granite-switch-vllm27') or (extra == 'group-14-granite-switch-dev-vllm27' and extra == 'group-14-granite-switch-dev-vllm30') or (extra == 'group-14-granite-switch-dev-vllm27' and extra == 'group-14-granite-switch-vllm30') or (extra == 'group-14-granite-switch-dev-vllm30' and extra == 'group-14-granite-switch-vllm27') or (extra == 'group-14-granite-switch-vllm27' and extra == 'group-14-granite-switch-vllm30')" },