Feat (brevitas_examples/vllm): evaluate LLM through vLLM

docsrc/source/user_guide/export_vllm.rst

@@ -171,15 +171,79 @@ pointing to the export directory:
 
 .. code-block:: python
 
+   # Register Brevitas' custom quantization method before constructing the engine.
+   import brevitas.export.inference.vLLM.manager
    from vllm import LLM
 
    llm = LLM(model="./exported_model", quantization="quant_brevitas")
 
-Or via the vLLM CLI:
+Zero-Shot Evaluation
+====================
+
+The ``brevitas_vllm_eval`` entrypoint evaluates an exported model through vLLM using either
+lm-evaluation-harness or LightEval. The latest published harnesses require incompatible vLLM
+versions, so install them in separate environments. For lm-evaluation-harness:
 
 .. code-block:: bash
 
-   vllm serve ./exported_model --quantization quant_brevitas
+   pip install -e ".[vllm_eval_lm_eval]"
+
+The evaluation backend must be selected explicitly. For lm-evaluation-harness:
+
+.. code-block:: bash
+
+   brevitas_vllm_eval --model ./exported_model --backend lm_eval
+
+In a separate environment, install and run LightEval with:
+
+.. code-block:: bash
+
+   pip install -e ".[vllm_eval_lighteval]"
+   brevitas_vllm_eval --model ./exported_model --backend lighteval
+
+Both commands evaluate ARC Challenge, ARC Easy, WinoGrande, and PIQA with zero-shot prompts by
+default. Use ``--tasks`` to override the task list. Task names follow the selected harness; bare
+LightEval task names are converted to zero-shot task specifications automatically. For example:
+
+.. code-block:: bash
+
+   brevitas_vllm_eval --model ./exported_model --backend lm_eval \
+      --tasks hellaswag piqa --tensor-parallel-size 2 --batch-size auto
+
+The CLI also exposes ``--dtype``, ``--gpu-memory-utilization``, ``--max-model-length``,
+``--max-new-tokens``, ``--limit``, ``--seed``, and ``--output-dir``. As with the LLM
+quantization entrypoint, these options can be supplied through a YAML file using ``--config``.
+The evaluator always loads the model with the ``quant_brevitas`` vLLM quantization method.
+The base Brevitas dependency keeps a broad Torch requirement; the vLLM version installed by each
+evaluation extra may impose a narrower Torch requirement in that environment.
+
+Thinking Models
+---------------
+
+Thinking is disabled by default. Likelihood and multiple-choice tasks always use plain-text
+prompts so an empty reasoning block or assistant prefix does not alter continuation scores.
+Generative tasks use the model chat template and explicitly disable thinking:
+
+.. code-block:: bash
+
+   brevitas_vllm_eval --model ./exported_model --backend lm_eval --tasks gsm8k \
+      --thinking disabled
+
+Enable thinking for generative tasks and configure the tags removed before metric computation
+with:
+
+.. code-block:: bash
+
+   brevitas_vllm_eval --model ./exported_model --backend lighteval --tasks gsm8k \
+      --thinking enabled --reasoning-start-tag '<think>' --reasoning-end-tag '</think>' \
+      --max-new-tokens 2048
+
+With lm-evaluation-harness, generative and likelihood tasks must be evaluated in separate
+invocations because chat-template and thinking settings apply to the entire evaluation. LightEval
+supports mixed task sets by selecting prompt formatting for each request type, provided an
+individual task document does not request both generation and likelihood metrics. The reasoning
+start and end tags are both used by LightEval. lm-evaluation-harness supports only an end token and
+therefore uses ``--reasoning-end-tag`` to remove the reasoning prefix.
 
 
 FAQ
@@ -204,5 +268,5 @@ the underlying quantization format maps to one of the supported inference handle
 * *Why do I get an import error for vLLM?*
 
 vLLM is not bundled with Brevitas and must be installed separately in your environment.
-The vLLM-specific code is only imported when ``--export-target vllm`` is specified, so
-vLLM is not required for other Brevitas workflows.
+The vLLM-specific code is imported only by the vLLM export and evaluation entrypoints, so vLLM
+is not required for other Brevitas workflows.

requirements/requirements-vllm-eval-lighteval.txt

@@ -0,0 +1,3 @@
+lighteval[math,vllm]==0.13.0
+# xxhash is a lighteval dependency; version 4 is incompatible with lighteval 0.13.
+xxhash<4

requirements/requirements-vllm-eval-lm-eval.txt

@@ -0,0 +1 @@
+lm-eval[vllm]==0.4.12

setup.py

@@ -42,6 +42,8 @@ def read_requirements(filename):
         "stt": read_requirements('requirements-stt.txt'),
         "llm": read_requirements('requirements-llm.txt'),
         "lighteval": read_requirements('requirements-lighteval.txt'),
+        "vllm_eval_lm_eval": read_requirements('requirements-vllm-eval-lm-eval.txt'),
+        "vllm_eval_lighteval": read_requirements('requirements-vllm-eval-lighteval.txt'),
         "diffusion": read_requirements('requirements-diffusion.txt'),
         "vision": read_requirements('requirements-vision.txt'),
         "finn_integration": read_requirements('requirements-finn-integration.txt'),
@@ -59,4 +61,5 @@ def read_requirements(filename):
             'brevitas_quartznet_preprocess = brevitas_examples.speech_to_text.get_librispeech_data:main',
             'brevitas_melgan_preprocess = brevitas_examples.text_to_speech.preprocess_dataset:main',
             'brevitas_ptq_imagenet_val = brevitas_examples.imagenet_classification.ptq.ptq_evaluate:main',
-            'brevitas_ptq_llm = brevitas_examples.llm.main:main'],})
+            'brevitas_ptq_llm = brevitas_examples.llm.main:main',
+            'brevitas_vllm_eval = brevitas_examples.llm.eval_vllm:main'],})

src/brevitas_examples/llm/eval_lighteval.py

@@ -25,7 +25,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from functools import partial
 import os
 import pathlib
 import re
@@ -39,12 +38,12 @@
 from lighteval.pipeline import Pipeline
 from lighteval.pipeline import PipelineParameters
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
-from lighteval.tasks.prompt_manager import PromptManager
 from lighteval.tasks.requests import Doc
-from lighteval.tasks.requests import SamplingMethod
 from torch import nn
 from transformers import AutoTokenizer
 
+from brevitas_examples.llm.lighteval_prompt import BrevitasPromptManager
+
 ### LightEval Custom Tasks
 
 # In most recent versions of lighteval, some tasks have been changed, differing from what lm_eval does
@@ -133,50 +132,6 @@ def piqa_harness(line, task_name: str = None):
 ### End of LightEval custom tasks
 
 
-class BrevitasPromptManager(PromptManager):
-    """Task-type-aware PromptManager that handles reasoning models like Qwen3.
-
-    Reasoning models (e.g. Qwen3) have two problems with lighteval's default PromptManager:
-
-    1. **Loglikelihood tasks**: When a chat template is used, Qwen3's template ends
-       the prompt with ``<|im_start|>assistant\n``, at which point the model's probability
-       distribution heavily favours ``<think>`` as the next token.  Passing
-       ``enable_thinking=False`` makes it worse by injecting an empty
-       ``<think>\\n\\n</think>\\n\\n`` block between context and continuation, corrupting
-       the loglikelihood computation.  Plain-text formatting avoids both issues.
-    2. **Generative tasks** (e.g. GSM8K): Instruct-tuned models need the chat template
-       to produce useful output, but thinking mode must be suppressed so the model does
-       not waste the token budget on ``<think>...</think>`` blocks.
-
-    This subclass inspects ``doc.sampling_methods`` and routes accordingly:
-
-    * ``LOGPROBS`` / ``PERPLEXITY`` → plain-text formatting (no chat template).
-    * ``GENERATIVE`` → chat template with ``enable_thinking=False``.
-
-    For non-reasoning models the ``enable_thinking`` kwarg is silently ignored by Jinja2,
-    so this is safe to use unconditionally.
-    """
-
-    def prepare_prompt(self, doc: Doc) -> str:
-        is_generative = SamplingMethod.GENERATIVE in doc.sampling_methods
-        if is_generative and self.use_chat_template:
-            return self._prepare_chat_template_no_thinking(doc)
-        else:
-            # For loglikelihood / perplexity tasks, always use plain text so
-            # that no thinking block or chat framing interferes with the
-            # probability computation over continuation tokens.
-            return self._prepare_plain_text(doc)
-
-    def _prepare_chat_template_no_thinking(self, doc: Doc) -> str:
-        """Format using the chat template with thinking mode explicitly disabled."""
-        orig_apply = self.tokenizer.apply_chat_template
-        try:
-            self.tokenizer.apply_chat_template = partial(orig_apply, enable_thinking=False)
-            return self._prepare_chat_template(doc)
-        finally:
-            self.tokenizer.apply_chat_template = orig_apply
-
-
 def filter_results(results, tasks):
     # filter out what we actually want to track
     eval_results = dict()