Add gamepad SE3/SE2 device: schema, tracker, plugin, retargeters

CMakeLists.txt

@@ -184,6 +184,7 @@ if(BUILD_PLUGINS)
 
     add_subdirectory(src/plugins/controller_se3_tracker)
     add_subdirectory(src/plugins/controller_synthetic_hands)
+    add_subdirectory(src/plugins/gamepad)
     add_subdirectory(src/plugins/generic_3axis_pedal)
     add_subdirectory(src/plugins/so101_leader)
     add_subdirectory(src/plugins/rebot_devarm_leader)

examples/teleop/python/gamepad_printer_example.py

@@ -0,0 +1,127 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Gamepad Printer Example.
+
+Prints every currently-held button and the full axis array each frame, via
+GamepadSource's "gamepad_buttons" and "gamepad_axes" outputs. Carries no semantic
+mapping (stick, trigger, toggle) -- that belongs in a retargeter (e.g.
+GamepadToSe3RelRetargeter) consuming this source's output. The gamepad plugin
+self-discovers its device and is auto-launched by TeleopSession -- no external
+process to start manually.
+"""
+
+import sys
+import time
+from pathlib import Path
+
+from isaacteleop.cloudxr import CloudXRLauncher
+from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource
+from isaacteleop.teleop_session_manager import (
+    TeleopSession,
+    TeleopSessionConfig,
+    PluginConfig,
+)
+
+
+PLUGIN_ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "plugins"
+PLUGIN_NAME = "gamepad"
+PLUGIN_ROOT_ID = "gamepad"
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    CloudXRLauncher.add_launcher_arguments(parser)
+    args = parser.parse_args()
+
+    print("\n" + "=" * 80)
+    print("  Gamepad Printer Example")
+    print("=" * 80)
+    print("Press any button or move a stick on the connected gamepad.")
+    print("=" * 80 + "\n")
+
+    # ==================================================================
+    # Setup: Create gamepad source
+    # ==================================================================
+    gamepad_source = GamepadSource(name="gamepad")
+
+    # ==================================================================
+    # Configure Plugins
+    # ==================================================================
+
+    plugins = []
+    if PLUGIN_ROOT_DIR.exists():
+        plugins.append(
+            PluginConfig(
+                plugin_name=PLUGIN_NAME,
+                plugin_root_id=PLUGIN_ROOT_ID,
+                search_paths=[PLUGIN_ROOT_DIR],
+            )
+        )
+
+    # ==================================================================
+    # Create and run TeleopSession
+    # ==================================================================
+
+    session_config = TeleopSessionConfig(
+        app_name="GamepadPrinterExample",
+        trackers=[],
+        pipeline=gamepad_source,
+        plugins=plugins,
+    )
+
+    with CloudXRLauncher.launch_context(args):
+        with TeleopSession(session_config) as session:
+            start_time = time.time()
+            prev_pressed: set[int] = set()
+
+            while time.time() - start_time < 30.0:
+                result = session.step()
+                buttons_group = result["gamepad_buttons"]
+                axes_group = result["gamepad_axes"]
+
+                elapsed = session.get_elapsed_time()
+                if buttons_group.is_none:
+                    print(
+                        f"[{elapsed:5.1f}s] (no gamepad data yet)",
+                        end="\r",
+                        flush=True,
+                    )
+                    time.sleep(0.01)
+                    continue
+
+                bitmap = buttons_group[0]
+                axes = axes_group[0]
+                pressed = {code for code in range(len(bitmap)) if bitmap[code]}
+                axes_str = " ".join(f"{v:+.2f}" for v in axes)
+
+                # Live status line (overwritten each frame).
+                names = [f"btn{code}" for code in sorted(pressed)]
+                print(
+                    f"[{elapsed:5.1f}s] Axes: [{axes_str}]  Held: {' '.join(names) or '-'}"
+                    + " " * 20,
+                    end="\r",
+                    flush=True,
+                )
+
+                # Permanent, scrollable log of every press/release transition -- a
+                # quick tap can flash by on the status line above before you notice
+                # it, but every transition is logged here.
+                for code in sorted(pressed - prev_pressed):
+                    print(f"[{elapsed:5.1f}s] btn{code} down")
+                for code in sorted(prev_pressed - pressed):
+                    print(f"[{elapsed:5.1f}s] btn{code} up")
+                prev_pressed = pressed
+
+                time.sleep(0.01)  # ~100 FPS
+
+            print("\nTime limit reached.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/core/deviceio_trackers/trackers.toml

@@ -28,6 +28,12 @@ traits = "PedalRecordingTraits"
 max_flatbuffer_size = 256
 python_accessor = "get_pedal_data"
 
+[[tracker]]
+name = "gamepad"
+table = "GamepadOutput"
+max_flatbuffer_size = 512
+python_accessor = "get_gamepad_data"
+
 [[tracker]]
 name = "haptic_command"
 direction = "push"

src/core/schema/fbs/gamepad.fbs

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+include "timestamp.fbs";
+
+namespace core;
+
+// Raw state of a Linux joystick-API gamepad (e.g. /dev/input/js0): the set of
+// currently-held button indices and the current value of every reported axis.
+// Carries no semantic mapping -- which axis/button means what (a stick, a trigger,
+// a toggle) is entirely up to the consuming retargeter.
+//
+// All fields are always present whenever this table itself is present.
+table GamepadOutput {
+    // Button indices (Linux joystick JS_EVENT_BUTTON numbers) currently held down,
+    // in no particular order.
+    pressed_buttons: [ushort] (id: 0);
+
+    // Current value of every reported axis (Linux joystick JS_EVENT_AXIS numbers),
+    // normalized to [-1, 1]. Index i is the value of axis i.
+    axes: [float] (id: 1);
+
+    // Whether the tracker has emitted at least one sample.
+    is_valid: bool (id: 2);
+}
+
+// MCAP recording wrapper for GamepadOutput.
+table GamepadOutputRecord {
+    data: GamepadOutput (id: 0);
+    timestamp: DeviceDataTimestamp (id: 1);
+}
+
+root_type GamepadOutputRecord;

src/core/schema/python/CMakeLists.txt

@@ -5,6 +5,7 @@ pybind11_add_module(schema_py
     oak_bindings.h
     controller_bindings.h
     full_body_bindings.h
+    gamepad_bindings.h
     hand_bindings.h
     haptic_command_bindings.h
     head_bindings.h

src/core/schema/python/gamepad_bindings.h

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Python bindings for the Gamepad FlatBuffer schema.
+// Types: GamepadOutput (table), exposed as an encoded view.
+
+#pragma once
+
+#include "schema_serialized.h"
+
+#include <pybind11/pybind11.h>
+#include <schema/gamepad_generated.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace py = pybind11;
+
+namespace core
+{
+
+inline void bind_gamepad(py::module& m)
+{
+    serialized_class<GamepadOutput>(m, "GamepadOutput", "Encoded raw joystick-API button/axis state.")
+        .def(py::init(
+                 [](std::vector<uint16_t> pressed_buttons, std::vector<float> axes, bool is_valid)
+                 {
+                     GamepadOutputT native;
+                     native.pressed_buttons = std::move(pressed_buttons);
+                     native.axes = std::move(axes);
+                     native.is_valid = is_valid;
+                     return pack<GamepadOutput>(native);
+                 }),
+             py::arg("pressed_buttons"), py::arg("axes"), py::arg("is_valid"), "Encode a gamepad button/axis snapshot.")
+        .def_property_readonly("pressed_buttons", vector_field(&GamepadOutput::pressed_buttons))
+        .def_property_readonly("axes", vector_field(&GamepadOutput::axes))
+        .def_property_readonly("is_valid", field(&GamepadOutput::is_valid))
+        .def("__repr__",
+             [](const Serialized<GamepadOutput>& self)
+             {
+                 std::string result = "GamepadOutput(pressed_buttons=[";
+                 const auto* buttons = self->pressed_buttons();
+                 if (buttons != nullptr)
+                 {
+                     for (size_t i = 0; i < buttons->size(); ++i)
+                     {
+                         if (i > 0)
+                             result += ", ";
+                         result += std::to_string((*buttons)[i]);
+                     }
+                 }
+                 result += "], axes=[";
+                 const auto* axes = self->axes();
+                 if (axes != nullptr)
+                 {
+                     for (size_t i = 0; i < axes->size(); ++i)
+                     {
+                         if (i > 0)
+                             result += ", ";
+                         result += std::to_string((*axes)[i]);
+                     }
+                 }
+                 result += "], is_valid=" + std::to_string(self->is_valid()) + ")";
+                 return result;
+             });
+
+    bind_record<GamepadOutputRecord, GamepadOutput>(m, "GamepadOutputRecord", "GamepadOutput");
+}
+
+} // namespace core

src/core/schema/python/schema_module.cpp

@@ -8,6 +8,7 @@
 // Include binding definitions.
 #include "controller_bindings.h"
 #include "full_body_bindings.h"
+#include "gamepad_bindings.h"
 #include "hand_bindings.h"
 #include "haptic_command_bindings.h"
 #include "head_bindings.h"
@@ -44,6 +45,9 @@ PYBIND11_MODULE(_schema, m)
     // Bind pedals types (Generic3AxisPedalOutput table).
     core::bind_pedals(m);
 
+    // Bind gamepad types (GamepadOutput table) for raw joystick-API button/axis state.
+    core::bind_gamepad(m);
+
     // Bind OGLO tactile glove types (OgloGloveSample table).
     core::bind_oglo_tactile(m);
 

src/plugins/gamepad/CMakeLists.txt

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    message(STATUS "Skipping gamepad plugin (Linux only)")
+    add_custom_target(gamepad_plugin
+        COMMAND ${CMAKE_COMMAND} -E echo "Skipping gamepad: Linux only")
+    return()
+endif()
+
+add_executable(gamepad_plugin
+    main.cpp
+    gamepad_plugin.cpp
+)
+
+target_link_libraries(gamepad_plugin PRIVATE
+    pusherio::pusherio
+    oxr::oxr_core
+    isaacteleop_schema
+)
+
+install(TARGETS gamepad_plugin RUNTIME DESTINATION plugins/gamepad)
+install(FILES plugin.yaml README.md DESTINATION plugins/gamepad)

src/plugins/gamepad/README.md

@@ -0,0 +1,46 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Gamepad Plugin
+
+Reads a gamepad from `/dev/input/js*` (Linux joystick API) and pushes `GamepadOutput` via OpenXR.
+Use with `GamepadTracker` with the same `collection_id`.
+
+Reports raw button/axis state only, with no semantic mapping to sticks, triggers, or commands --
+that mapping belongs in a retargeter (e.g. `GamepadToSe3RelRetargeter`) consuming this tracker's
+output.
+
+Self-discovers its device (the first `*-joystick` entry under `/dev/input/by-path/`), so it needs
+no arguments to run and can be auto-launched by `PluginManager` via `PluginConfig` -- no manual
+process to start.
+
+## Usage
+
+Auto-launched (recommended -- matches how `PluginManager` invokes plugins):
+
+```bash
+./gamepad_plugin --plugin-root-id=gamepad
+```
+
+Manual / standalone, with an explicit device:
+
+```bash
+./gamepad_plugin [device_path] [--plugin-root-id=<collection_id>]
+```
+
+- **device_path**: Optional. Defaults to the first `*-joystick` entry under `/dev/input/by-path/`.
+  Identify a specific gamepad with `cat /proc/bus/input/devices` (look for a `Handlers=... jsN`
+  line under a gamepad entry) or `jstest /dev/input/jsN`. Reading `/dev/input/js*` typically
+  requires membership in the `input` group.
+- **collection_id**: Default `gamepad`. Match this when creating `GamepadTracker`.
+
+## Button/axis mapping
+
+Reports every axis value (normalized to `[-1, 1]`) and the set of currently-held button indices,
+as reported by the Linux joystick API (`JS_EVENT_AXIS` / `JS_EVENT_BUTTON`, see
+`linux/joystick.h`). Axis/button indices and count depend on the connected device's driver (e.g.
+`xpad` for Xbox-style controllers) -- no fixed mapping is assumed here.
+
+Linux only.