From db2d60b1a5d5af4e49949483c6dcff3ec820fa0e Mon Sep 17 00:00:00 2001 From: jhcipar Date: Mon, 17 Nov 2025 10:04:14 -0500 Subject: [PATCH 1/3] feat: hello world example --- .../01_hello_world/.flashignore | 40 +++++++ .../01_hello_world/gpu_worker.py | 109 ++++++++++++++++++ 01_getting_started/01_hello_world/main.py | 75 ++++++++++++ 01_getting_started/README.md | 5 +- 4 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 01_getting_started/01_hello_world/.flashignore create mode 100644 01_getting_started/01_hello_world/gpu_worker.py create mode 100644 01_getting_started/01_hello_world/main.py diff --git a/01_getting_started/01_hello_world/.flashignore b/01_getting_started/01_hello_world/.flashignore new file mode 100644 index 0000000..ea5988c --- /dev/null +++ b/01_getting_started/01_hello_world/.flashignore @@ -0,0 +1,40 @@ +# Flash Build Ignore Patterns + +# Python cache +__pycache__/ +*.pyc + +# Virtual environments +venv/ +.venv/ +env/ + +# IDE +.vscode/ +.idea/ + +# Environment files +.env +.env.local + +# Git +.git/ +.gitignore + +# Build artifacts +dist/ +build/ +*.egg-info/ + +# Flash resources +.tetra_resources.pkl + +# Tests +tests/ +test_*.py +*_test.py + +# Documentation +docs/ +*.md +!README.md diff --git a/01_getting_started/01_hello_world/gpu_worker.py b/01_getting_started/01_hello_world/gpu_worker.py new file mode 100644 index 0000000..fefb74b --- /dev/null +++ b/01_getting_started/01_hello_world/gpu_worker.py @@ -0,0 +1,109 @@ +## Hello world: GPU serverless workers +# In this part of the example code, we provision a GPU-based worker and have it +# execute code. We can run the worker directly, or have it handle API requests +# to the router function. It's registered to a subrouter in the __init__.py +# file in this folder, and subsequently imported by main.py and attached to the +# FastAPI app there. + +# Scaling behavior is controlled by configuration passed to the +# `LiveServerless` class. +from tetra_rp import ( + GpuGroup, + LiveServerless, + remote, +) + +from fastapi import APIRouter +from pydantic import BaseModel + + +# Here, we'll define several variables that change the +# default behavior of our serverless endpoint. `workersMin` sets our endpoint +# to scale to 0 active containers; `workersMax` will allow our endpoint to run +# up to 3 workers in parallel as the endpoint receives more work. We also set +# an idle timeout of 5 minutes so that any active worker stays alive for 5 +# minutes after completing a request. +gpu_config = LiveServerless( + name="gpu_worker", + gpus=[GpuGroup.ANY], # Run on any GPU + workersMin=0, + workersMax=3, + idleTimeout=5, +) + + +# Decorating our function with `remote` will package up the function code and +# deploy it on the infrastructure according to the passed input config. The +# results from the worker will be returned to your terminal. In this example +# the function will return a greeting to the input string passed in the `name` +# key. The code itself will run on a GPU worker, and information about the GPU +# the worker has access to will be included in the response. +@remote(resource_config=gpu_config) +async def gpu_hello( + input_data: dict, +) -> dict: + """Simple GPU worker example with GPU detection.""" + import platform + from datetime import datetime + + import torch + + gpu_available = torch.cuda.is_available() + gpu_name = torch.cuda.get_device_name(0) + gpu_count = torch.cuda.device_count() + gpu_memory = torch.cuda.get_device_properties(0).total_memory / (1024**3) + + message = input_data.get( + "message", + "Hello from GPU worker!", + ) + + return { + "status": "success", + "message": message, + "worker_type": "GPU", + "gpu_info": { + "available": gpu_available, + "name": gpu_name, + "count": gpu_count, + "memory_gb": round( + gpu_memory, + 2, + ), + }, + "timestamp": datetime.now().isoformat(), + "platform": platform.system(), + "python_version": platform.python_version(), + } + + +# We define a subrouter for our gpu worker so that our main router in `main.py` +# can attach it for routing gpu-specific requests. +gpu_router = APIRouter() + + +class MessageRequest(BaseModel): + """Request model for GPU worker.""" + + message: str = "Hello from GPU!" + + +@gpu_router.post("/hello") +async def hello( + request: MessageRequest, +): + """Simple GPU worker endpoint.""" + result = await gpu_hello({"message": request.message}) + return result + + +# This code is packaged up as a "worker" that will handle requests sent to the +# endpoint at /gpu/hello, but you can also trigger it directly by running +# python -m workers.gpu.endpoint +if __name__ == "__main__": + import asyncio + + test_payload = {"message": "Testing GPU worker"} + print(f"Testing GPU worker with payload: {test_payload}") + result = asyncio.run(gpu_hello(test_payload)) + print(f"Result: {result}") diff --git a/01_getting_started/01_hello_world/main.py b/01_getting_started/01_hello_world/main.py new file mode 100644 index 0000000..e624254 --- /dev/null +++ b/01_getting_started/01_hello_world/main.py @@ -0,0 +1,75 @@ +## Example 1: Hello world +# This is an example of a simple Flash application. +# It consists of an API router (this file) that routes requests to a local +# endpoint on your machine to worker code. Worker code is executed on Runpod +# infrastructure on gpu and cpu-based serverless workers. + +# Subrouters and associated worker function code are defined in the ./workers/ +# dir and attached to the main router in this file. By default, running +# `flash run` will start the local API server on your machine serving requests +# from port 8888. + +# We'll define a resource configuration for a GPU worker on a Runpod serverless +# endpoint. The GPU worker will return information about the infrastructure +# it executes on. + +import logging +import os + +from fastapi import FastAPI +from gpu_worker import gpu_router + +logger = logging.getLogger(__name__) + +# We define a simple FastAPI app to serve requests from localhost. +app = FastAPI( + title="Flash Application", + description="Distributed GPU and CPU computing with Runpod Flash", + version="0.1.0", +) + +# Attach gpu and cpu worker subrouters - this will route any requests to our +# app with the prefix /gpu and /cpu to get sent to the gpu and cpu subrouters, +# respectively. For example, curl -X POST http://localhost:{PORT}/cpu will be +# handled entirely by the cpu subrouter. +app.include_router( + gpu_router, + prefix="/gpu", + tags=["GPU Workers"], +) + + +# The homepage for our main endpoint will just return a plaintext json object +# containing the endpoints defined in this app. +@app.get("/") +def home(): + return { + "message": "Flash Application", + "docs": "/docs", + "endpoints": { + "gpu_hello": "/gpu/hello", + }, + } + + +@app.get("/ping") +def ping(): + return {"status": "healthy"} + + +if __name__ == "__main__": + import uvicorn + + port = int( + os.getenv( + "PORT", + 8888, + ) + ) + logger.info(f"Starting Flash server on port {port}") + + uvicorn.run( + app, + host="0.0.0.0", + port=port, + ) diff --git a/01_getting_started/README.md b/01_getting_started/README.md index 7201b32..c130139 100644 --- a/01_getting_started/README.md +++ b/01_getting_started/README.md @@ -5,18 +5,17 @@ Fundamental concepts for building Flash applications. Start here if you're new t ## Examples ### [01_hello_world](./01_hello_world/) -The simplest Flash application with GPU and CPU workers. +The simplest Flash application with GPU workers **What you'll learn:** - Basic Flash application structure -- Creating GPU and CPU workers +- Creating GPU workers - Using the `@remote` decorator - Running Flash applications locally - Testing endpoints with FastAPI docs **Concepts:** - `LiveServerless` configuration for GPU workers -- `CpuLiveServerless` configuration for CPU workers - Worker auto-scaling (min/max workers) - FastAPI router integration From d6e1ac7ef35ffaf89a7b0123927de6db5c321174 Mon Sep 17 00:00:00 2001 From: jhcipar Date: Mon, 17 Nov 2025 15:32:53 -0500 Subject: [PATCH 2/3] format: ruff --- 01_getting_started/01_hello_world/gpu_worker.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/01_getting_started/01_hello_world/gpu_worker.py b/01_getting_started/01_hello_world/gpu_worker.py index fefb74b..14641b6 100644 --- a/01_getting_started/01_hello_world/gpu_worker.py +++ b/01_getting_started/01_hello_world/gpu_worker.py @@ -7,16 +7,15 @@ # Scaling behavior is controlled by configuration passed to the # `LiveServerless` class. +from fastapi import APIRouter +from pydantic import BaseModel + from tetra_rp import ( GpuGroup, LiveServerless, remote, ) -from fastapi import APIRouter -from pydantic import BaseModel - - # Here, we'll define several variables that change the # default behavior of our serverless endpoint. `workersMin` sets our endpoint # to scale to 0 active containers; `workersMax` will allow our endpoint to run From 26c5ce5900f99afd18d0e2ce12756405eeaf923a Mon Sep 17 00:00:00 2001 From: jhcipar Date: Mon, 17 Nov 2025 15:40:29 -0500 Subject: [PATCH 3/3] cleanup: gpu workers only --- 01_getting_started/01_hello_world/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/01_getting_started/01_hello_world/main.py b/01_getting_started/01_hello_world/main.py index e624254..3d2f0a9 100644 --- a/01_getting_started/01_hello_world/main.py +++ b/01_getting_started/01_hello_world/main.py @@ -2,7 +2,7 @@ # This is an example of a simple Flash application. # It consists of an API router (this file) that routes requests to a local # endpoint on your machine to worker code. Worker code is executed on Runpod -# infrastructure on gpu and cpu-based serverless workers. +# infrastructure on serverless workers with GPUs. # Subrouters and associated worker function code are defined in the ./workers/ # dir and attached to the main router in this file. By default, running @@ -24,14 +24,14 @@ # We define a simple FastAPI app to serve requests from localhost. app = FastAPI( title="Flash Application", - description="Distributed GPU and CPU computing with Runpod Flash", + description="Distributed GPU computing with Runpod Flash", version="0.1.0", ) -# Attach gpu and cpu worker subrouters - this will route any requests to our -# app with the prefix /gpu and /cpu to get sent to the gpu and cpu subrouters, -# respectively. For example, curl -X POST http://localhost:{PORT}/cpu will be -# handled entirely by the cpu subrouter. +# Attach gpu worker subrouters - this will route any requests to our +# app with the prefix /gpu to the gpu subrouter. To see the subrouter in action, +# start the app and execute the following command in another terminal window: +# curl -X POST http://localhost:8888/gpu/hello -d '{"input": "hello"}' -H "Content-Type: application/json" app.include_router( gpu_router, prefix="/gpu",