Add gamepad SE3/SE2 device: schema, tracker, plugin, retargeters (1.4.x

CMakeLists.txt

@@ -186,6 +186,7 @@ if(BUILD_PLUGINS)
 
     add_subdirectory(src/plugins/controller_se3_tracker)
     add_subdirectory(src/plugins/controller_synthetic_hands)
+    add_subdirectory(src/plugins/gamepad)
     add_subdirectory(src/plugins/generic_3axis_pedal)
     add_subdirectory(src/plugins/so101_leader)
     add_subdirectory(src/plugins/rebot_devarm_leader)
@@ -199,5 +200,23 @@ if(BUILD_PLUGINS)
     endif()
 endif()
 
+# Bundle the gamepad plugin binary + metadata inside the wheel (isaacteleop.plugins.
+# gamepad) so `pip install isaacteleop` alone is sufficient to run it via
+# PluginManager -- no separate `cmake --install` step required. A standalone target
+# (rather than baked into python_package's own COMMAND list in
+# src/core/python/CMakeLists.txt) because src/core is added before src/plugins, so
+# `gamepad_plugin` isn't defined yet at that point in the configure.
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND TARGET gamepad_plugin AND TARGET python_package)
+    add_custom_target(bundle_gamepad_plugin ALL
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$<CONFIG>/isaacteleop/plugins/gamepad"
+        COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/src/core/python/isaacteleop_plugins_init.py" "${CMAKE_BINARY_DIR}/python_package/$<CONFIG>/isaacteleop/plugins/__init__.py"
+        COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/src/core/python/isaacteleop_plugins_gamepad_init.py" "${CMAKE_BINARY_DIR}/python_package/$<CONFIG>/isaacteleop/plugins/gamepad/__init__.py"
+        COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/src/plugins/gamepad/plugin.yaml" "${CMAKE_BINARY_DIR}/python_package/$<CONFIG>/isaacteleop/plugins/gamepad/plugin.yaml"
+        COMMAND ${CMAKE_COMMAND} -E copy "$<TARGET_FILE:gamepad_plugin>" "${CMAKE_BINARY_DIR}/python_package/$<CONFIG>/isaacteleop/plugins/gamepad/gamepad_plugin"
+        DEPENDS gamepad_plugin python_package
+        COMMENT "Bundling gamepad plugin into the Python package"
+    )
+endif()
+
 # Formatting enforcement (runs on Linux by default)
 include(cmake/ClangFormat.cmake)

examples/teleop/python/gamepad_printer_example.py

@@ -0,0 +1,127 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Gamepad Printer Example.
+
+Prints every currently-held button and the full axis array each frame, via
+GamepadSource's "gamepad_buttons" and "gamepad_axes" outputs. Carries no semantic
+mapping (stick, trigger, toggle) -- that belongs in a retargeter (e.g.
+GamepadToSe3RelRetargeter) consuming this source's output. The gamepad plugin
+self-discovers its device and is auto-launched by TeleopSession -- no external
+process to start manually.
+"""
+
+import sys
+import time
+
+from isaacteleop.cloudxr import CloudXRLauncher
+from isaacteleop.plugins import plugin_search_path
+from isaacteleop.retargeting_engine.deviceio_source_nodes import GamepadSource
+from isaacteleop.teleop_session_manager import (
+    TeleopSession,
+    TeleopSessionConfig,
+    PluginConfig,
+)
+
+
+PLUGIN_ROOT_DIR = plugin_search_path()
+PLUGIN_NAME = "gamepad"
+PLUGIN_ROOT_ID = "gamepad"
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    CloudXRLauncher.add_launcher_arguments(parser)
+    args = parser.parse_args()
+
+    print("\n" + "=" * 80)
+    print("  Gamepad Printer Example")
+    print("=" * 80)
+    print("Press any button or move a stick on the connected gamepad.")
+    print("=" * 80 + "\n")
+
+    # ==================================================================
+    # Setup: Create gamepad source
+    # ==================================================================
+    gamepad_source = GamepadSource(name="gamepad")
+
+    # ==================================================================
+    # Configure Plugins
+    # ==================================================================
+
+    plugins = []
+    if PLUGIN_ROOT_DIR.exists():
+        plugins.append(
+            PluginConfig(
+                plugin_name=PLUGIN_NAME,
+                plugin_root_id=PLUGIN_ROOT_ID,
+                search_paths=[PLUGIN_ROOT_DIR],
+            )
+        )
+
+    # ==================================================================
+    # Create and run TeleopSession
+    # ==================================================================
+
+    session_config = TeleopSessionConfig(
+        app_name="GamepadPrinterExample",
+        trackers=[],
+        pipeline=gamepad_source,
+        plugins=plugins,
+    )
+
+    with CloudXRLauncher.launch_context(args):
+        with TeleopSession(session_config) as session:
+            start_time = time.time()
+            prev_pressed: set[int] = set()
+
+            while time.time() - start_time < 30.0:
+                result = session.step()
+                buttons_group = result["gamepad_buttons"]
+                axes_group = result["gamepad_axes"]
+
+                elapsed = session.get_elapsed_time()
+                if buttons_group.is_none:
+                    print(
+                        f"[{elapsed:5.1f}s] (no gamepad data yet)",
+                        end="\r",
+                        flush=True,
+                    )
+                    time.sleep(0.01)
+                    continue
+
+                bitmap = buttons_group[0]
+                axes = axes_group[0]
+                pressed = {code for code in range(len(bitmap)) if bitmap[code]}
+                axes_str = " ".join(f"{v:+.2f}" for v in axes)
+
+                # Live status line (overwritten each frame).
+                names = [f"btn{code}" for code in sorted(pressed)]
+                print(
+                    f"[{elapsed:5.1f}s] Axes: [{axes_str}]  Held: {' '.join(names) or '-'}"
+                    + " " * 20,
+                    end="\r",
+                    flush=True,
+                )
+
+                # Permanent, scrollable log of every press/release transition -- a
+                # quick tap can flash by on the status line above before you notice
+                # it, but every transition is logged here.
+                for code in sorted(pressed - prev_pressed):
+                    print(f"[{elapsed:5.1f}s] btn{code} down")
+                for code in sorted(prev_pressed - pressed):
+                    print(f"[{elapsed:5.1f}s] btn{code} up")
+                prev_pressed = pressed
+
+                time.sleep(0.01)  # ~100 FPS
+
+            print("\nTime limit reached.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/core/deviceio_base/cpp/inc/deviceio_base/gamepad_tracker_base.hpp

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "tracker.hpp"
+
+namespace core
+{
+
+struct GamepadOutputTrackedT;
+
+// Abstract base interface for GamepadTracker implementations.
+class IGamepadTrackerImpl : public ITrackerImpl
+{
+public:
+    virtual const GamepadOutputTrackedT& get_data() const = 0;
+};
+
+} // namespace core

src/core/deviceio_trackers/cpp/CMakeLists.txt

@@ -10,6 +10,7 @@ add_library(deviceio_trackers STATIC
     controller_tracker.cpp
     message_channel_tracker.cpp
     generic_3axis_pedal_tracker.cpp
+    gamepad_tracker.cpp
     oglo_tactile_tracker.cpp
     tensor_push_tracker.cpp
     haptic_command_reader_tracker.cpp
@@ -23,6 +24,7 @@ add_library(deviceio_trackers STATIC
     inc/deviceio_trackers/message_channel_tracker.hpp
     inc/deviceio_trackers/full_body_tracker.hpp
     inc/deviceio_trackers/generic_3axis_pedal_tracker.hpp
+    inc/deviceio_trackers/gamepad_tracker.hpp
     inc/deviceio_trackers/oglo_tactile_tracker.hpp
     inc/deviceio_trackers/tensor_push_tracker.hpp
     inc/deviceio_trackers/haptic_command_reader_tracker.hpp

src/core/deviceio_trackers/cpp/gamepad_tracker.cpp

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "inc/deviceio_trackers/gamepad_tracker.hpp"
+
+namespace core
+{
+
+// ============================================================================
+// GamepadTracker
+// ============================================================================
+
+GamepadTracker::GamepadTracker(const std::string& collection_id, size_t max_flatbuffer_size)
+    : collection_id_(collection_id), max_flatbuffer_size_(max_flatbuffer_size)
+{
+}
+
+const GamepadOutputTrackedT& GamepadTracker::get_data(const ITrackerSession& session) const
+{
+    return static_cast<const IGamepadTrackerImpl&>(session.get_tracker_impl(*this)).get_data();
+}
+
+} // namespace core

src/core/deviceio_trackers/cpp/inc/deviceio_trackers/gamepad_tracker.hpp

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <deviceio_base/gamepad_tracker_base.hpp>
+#include <schema/gamepad_generated.h>
+
+#include <cstddef>
+#include <string>
+
+namespace core
+{
+
+/*!
+ * @brief Facade for raw joystick-API button/axis state exposed as ``GamepadOutputTrackedT``.
+ *
+ * Semantic contract: ``pressed_buttons`` is the set of currently-held Linux joystick
+ * button indices, ``axes`` is the current value of every reported axis; no semantic
+ * mapping (stick, trigger, toggle, ...) is applied here. After each
+ * ``ITrackerSession::update()`` that includes this tracker, ``get_data(session)``
+ * reflects the implementation's tracked snapshot. **Absent** data (``data`` null)
+ * means no sample has been unpacked yet or the collection/source is unavailable.
+ *
+ * Usage:
+ * @code
+ * auto tracker = std::make_shared<GamepadTracker>("gamepad");
+ * // ... register the tracker with a session, then each tick: ...
+ * session->update();
+ * const auto& data = tracker->get_data(*session);
+ * @endcode
+ */
+class GamepadTracker : public ITracker
+{
+public:
+    //! Default maximum FlatBuffer size for GamepadOutput messages.
+    static constexpr size_t DEFAULT_MAX_FLATBUFFER_SIZE = 512;
+
+    /*!
+     * @brief Constructs a GamepadTracker.
+     * @param collection_id Logical stream identifier; must match the data source for the chosen backend
+     *        (see live implementation documentation).
+     * @param max_flatbuffer_size Upper bound for serialized ``GamepadOutput`` / record payloads
+     *        (default: 512 bytes); must be sufficient for the schema and backend.
+     */
+    explicit GamepadTracker(const std::string& collection_id, size_t max_flatbuffer_size = DEFAULT_MAX_FLATBUFFER_SIZE);
+
+    std::string_view get_name() const override
+    {
+        return TRACKER_NAME;
+    }
+
+    /*!
+     * @brief Gamepad snapshot from the session's implementation.
+     *
+     * ``tracked.data`` is null when there is no valid last-known sample (source never
+     * provided data or implementation cleared state when the collection is gone).
+     */
+    const GamepadOutputTrackedT& get_data(const ITrackerSession& session) const;
+
+    const std::string& collection_id() const
+    {
+        return collection_id_;
+    }
+
+    size_t max_flatbuffer_size() const
+    {
+        return max_flatbuffer_size_;
+    }
+
+private:
+    static constexpr const char* TRACKER_NAME = "GamepadTracker";
+
+    std::string collection_id_;
+    size_t max_flatbuffer_size_;
+};
+
+} // namespace core

src/core/deviceio_trackers/python/deviceio_trackers_init.py

@@ -14,6 +14,7 @@
     MessageChannelTracker,
     FrameMetadataTrackerOak,
     Generic3AxisPedalTracker,
+    GamepadTracker,
     OgloTactileTracker,
     TensorPushTracker,
     JointStateTracker,
@@ -52,6 +53,7 @@ def __getattr__(name: str):
     "FrameMetadataTrackerOak",
     "FullBodyTracker",
     "Generic3AxisPedalTracker",
+    "GamepadTracker",
     "OgloTactileTracker",
     "TensorPushTracker",
     "JointStateTracker",

src/core/deviceio_trackers/python/tracker_bindings.cpp

@@ -4,6 +4,7 @@
 #include <deviceio_trackers/controller_tracker.hpp>
 #include <deviceio_trackers/frame_metadata_tracker_oak.hpp>
 #include <deviceio_trackers/full_body_tracker.hpp>
+#include <deviceio_trackers/gamepad_tracker.hpp>
 #include <deviceio_trackers/generic_3axis_pedal_tracker.hpp>
 #include <deviceio_trackers/hand_tracker.hpp>
 #include <deviceio_trackers/head_tracker.hpp>
@@ -154,6 +155,16 @@ PYBIND11_MODULE(_deviceio_trackers, m)
             { return self.get_data(session); },
             py::arg("session"), "Get the current foot pedal tracked state (data is None when no data available)");
 
+    py::class_<core::GamepadTracker, core::ITracker, std::shared_ptr<core::GamepadTracker>>(m, "GamepadTracker")
+        .def(py::init<const std::string&, size_t>(), py::arg("collection_id"),
+             py::arg("max_flatbuffer_size") = core::GamepadTracker::DEFAULT_MAX_FLATBUFFER_SIZE,
+             "Construct a GamepadTracker for the given tensor collection ID")
+        .def(
+            "get_gamepad_data",
+            [](const core::GamepadTracker& self, const core::ITrackerSession& session) -> core::GamepadOutputTrackedT
+            { return self.get_data(session); },
+            py::arg("session"), "Get the current gamepad tracked state (data is None when no data available)");
+
     py::class_<core::OgloTactileTracker, core::ITracker, std::shared_ptr<core::OgloTactileTracker>>(
         m, "OgloTactileTracker")
         .def(py::init<const std::string&, size_t>(), py::arg("collection_id"),

src/core/live_trackers/cpp/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(live_trackers STATIC
     live_message_channel_tracker_impl.cpp
     live_full_body_tracker_pico_impl.cpp
     live_generic_3axis_pedal_tracker_impl.cpp
+    live_gamepad_tracker_impl.cpp
     live_oglo_tactile_tracker_impl.cpp
     live_tensor_push_tracker_impl.cpp
     live_haptic_command_reader_tracker_impl.cpp
@@ -27,6 +28,7 @@ add_library(live_trackers STATIC
     live_message_channel_tracker_impl.hpp
     live_full_body_tracker_pico_impl.hpp
     live_generic_3axis_pedal_tracker_impl.hpp
+    live_gamepad_tracker_impl.hpp
     live_oglo_tactile_tracker_impl.hpp
     live_tensor_push_tracker_impl.hpp
     live_haptic_command_reader_tracker_impl.hpp

src/core/live_trackers/cpp/inc/live_trackers/live_deviceio_factory.hpp

@@ -33,6 +33,8 @@ class FullBodyTracker;
 class IFullBodyTrackerImpl;
 class Generic3AxisPedalTracker;
 class IGeneric3AxisPedalTrackerImpl;
+class GamepadTracker;
+class IGamepadTrackerImpl;
 class OgloTactileTracker;
 class IOgloTactileTrackerImpl;
 class TensorPushTracker;
@@ -91,6 +93,7 @@ class LiveDeviceIOFactory
     std::unique_ptr<IFullBodyTrackerImpl> create_full_body_tracker_pico_impl(const FullBodyTracker* tracker);
     std::unique_ptr<IGeneric3AxisPedalTrackerImpl> create_generic_3axis_pedal_tracker_impl(
         const Generic3AxisPedalTracker* tracker);
+    std::unique_ptr<IGamepadTrackerImpl> create_gamepad_tracker_impl(const GamepadTracker* tracker);
     std::unique_ptr<IOgloTactileTrackerImpl> create_oglo_tactile_tracker_impl(const OgloTactileTracker* tracker);
     std::unique_ptr<ITensorPushTrackerImpl> create_tensor_push_tracker_impl(const TensorPushTracker* tracker);
     std::unique_ptr<IHapticCommandReaderTrackerImpl> create_haptic_command_reader_tracker_impl(