From 3ea5e8d46025c18fb111769e99862d8fbb208a2a Mon Sep 17 00:00:00 2001 From: nissrin2020ali-ux Date: Fri, 28 Aug 2026 06:46:08 +0000 Subject: [PATCH] feat(serving): add OrcaRouter as a named LLM provider Adds APIOrcaRouterServing, an OpenAI-compatible serving class backed by the OrcaRouter gateway, alongside the existing APILLMServing_request. It reuses the parent's request/retry/formatting logic and only wires in OrcaRouter defaults (base URL, ORCAROUTER_API_KEY, orcarouter/auto), so DataFlow users can use the gateway as a first-class provider instead of an anonymous custom base URL. Includes a README subsection and unit tests mirroring the existing API serving test pattern. Co-Authored-By: Claude --- README.md | 23 ++++++++ dataflow/serving/__init__.py | 2 + dataflow/serving/api_orca_router_serving.py | 55 +++++++++++++++++++ test/cpu_only/test_api_orca_router_serving.py | 45 +++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 dataflow/serving/api_orca_router_serving.py create mode 100644 test/cpu_only/test_api_orca_router_serving.py diff --git a/README.md b/README.md index 050a7f56..58d7f984 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,29 @@ After running, the operator will append the generated results into output_key. F ] ``` +### 🧠 6.1 Named LLM Backends + +Besides pointing a serving class at any OpenAI-compatible endpoint, DataFlow ships a few +pre-configured named backends. For example, `APIOrcaRouterServing` wires the operator +stack to the [OrcaRouter](https://www.orcarouter.ai) gateway — an OpenAI-compatible AI +gateway that, like OpenRouter, exposes a provider/model namespace across many models +through a single endpoint, while also adding adaptive routing, automatic failover, +zero-markup inference, observability, guardrails, and agent-tool governance behind the +same endpoint: + +```python +from dataflow.serving import APIOrcaRouterServing + +# configure LLM serving with the OrcaRouter gateway +# api key needs to be set via `export ORCAROUTER_API_KEY=sk-orca-...` +llm_serving = APIOrcaRouterServing(model_name="orcarouter/auto") + +prompted_generator = PromptedGenerator( + llm_serving=llm_serving, # pre-configured LLM backend + system_prompt="Please solve this math problem." +) +``` +

🛠️ 7. Pipelines (Click to expand)

 diff --git a/dataflow/serving/__init__.py b/dataflow/serving/__init__.py index df759981..d2861d24 100644 --- a/dataflow/serving/__init__.py +++ b/dataflow/serving/__init__.py @@ -1,4 +1,5 @@ from .api_llm_serving_request import APILLMServing_request +from .api_orca_router_serving import APIOrcaRouterServing from .local_model_llm_serving import LocalModelLLMServing_vllm from .local_model_llm_serving import LocalModelLLMServing_sglang from .api_vlm_serving_openai import APIVLMServing_openai @@ -18,6 +19,7 @@ __all__ = [ "APIGoogleVertexAIServing", "APILLMServing_request", + "APIOrcaRouterServing", "LocalModelLLMServing_vllm", "LocalModelLLMServing_sglang", "APIVLMServing_openai", diff --git a/dataflow/serving/api_orca_router_serving.py b/dataflow/serving/api_orca_router_serving.py new file mode 100644 index 00000000..9f751216 --- /dev/null +++ b/dataflow/serving/api_orca_router_serving.py @@ -0,0 +1,55 @@ +from .api_llm_serving_request import APILLMServing_request + + +class APIOrcaRouterServing(APILLMServing_request): + """ + OpenAI-compatible serving class backed by the OrcaRouter gateway. + + OrcaRouter (https://www.orcarouter.ai) is an OpenAI-compatible AI gateway that, + like OpenRouter, exposes a provider/model namespace across many models through a + single endpoint. On top of that it adds adaptive routing, automatic failover, + zero-markup inference, observability, guardrails, and agent-tool governance on + the same OpenAI-compatible API. + + This class reuses the request/retry/formatting logic of APILLMServing_request and + only wires it to OrcaRouter defaults, so DataFlow users can adopt the gateway + without treating it as an anonymous custom base URL. + """ + def __init__(self, + api_url: str = "https://api.orcarouter.ai/v1/chat/completions", + key_name_of_api_key: str = "ORCAROUTER_API_KEY", + model_name: str = "orcarouter/auto", + temperature: float = 0.0, + max_workers: int = 10, + max_retries: int = 5, + connect_timeout: float = 10.0, + read_timeout: float = 120.0, + **configs: dict): + """ + Initialize OrcaRouter serving instance. + + Args: + api_url: OrcaRouter OpenAI-compatible chat completions endpoint + key_name_of_api_key: Environment variable holding the OrcaRouter API key + model_name: OrcaRouter model namespace id (e.g. "orcarouter/auto") + temperature: Sampling temperature + max_workers: Number of concurrent workers for batch processing + max_retries: Number of LLM inference retry chances for each input + connect_timeout: Connection timeout in seconds + read_timeout: Read timeout in seconds + **configs: Additional parameters forwarded to the API payload + + Note: + Set the API key via `export ORCAROUTER_API_KEY=sk-orca-...` before use. + """ + super().__init__( + api_url=api_url, + key_name_of_api_key=key_name_of_api_key, + model_name=model_name, + temperature=temperature, + max_workers=max_workers, + max_retries=max_retries, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + **configs, + ) diff --git a/test/cpu_only/test_api_orca_router_serving.py b/test/cpu_only/test_api_orca_router_serving.py new file mode 100644 index 00000000..e99bc997 --- /dev/null +++ b/test/cpu_only/test_api_orca_router_serving.py @@ -0,0 +1,45 @@ +import pytest +from dataflow.serving import APIOrcaRouterServing + + +@pytest.mark.api +def test_orca_router_serving_defaults_and_request(dummy_server_base_url, monkeypatch): + monkeypatch.setenv("ORCAROUTER_API_KEY", "dummy-key") + + api_url = ( + f"{dummy_server_base_url}/v1/chat/completions" + f"?queue=0&ka_interval=0.05&stream=0" + f"&body=hello&think=" + ) + + cli = APIOrcaRouterServing( + api_url=api_url, + model_name="orcarouter/auto", + connect_timeout=1.0, + read_timeout=3.0, + max_retries=1, + max_workers=1, + ) + + assert cli.api_url == api_url + assert cli.model_name == "orcarouter/auto" + assert cli.api_key == "dummy-key" + + _id, resp = cli._api_chat_with_id( + id=0, + payload=[{"role": "user", "content": "hi"}], + model="orcarouter/auto", + is_embedding=False, + ) + + assert _id == 0 + assert resp == "hello" + + cli.cleanup() + + +@pytest.mark.api +def test_orca_router_serving_requires_key(monkeypatch): + monkeypatch.delenv("ORCAROUTER_API_KEY", raising=False) + with pytest.raises(ValueError): + APIOrcaRouterServing()