Directly generate build script in recipe folder, and support additional_cmake_args and additional_folder arguments (#84)

* Generate build scripts and activation scripts directly in recipe directory without intermediate files

* Add support for additional_cmake_args in pkg_additional_info.yaml

* Add support for specifying additional_folder in rosdistro_additional_recipes.yaml

Author: traversaro

vinca/distro.py

@@ -153,7 +153,10 @@ def get_package_xml_for_additional_package(self, pkg_info):
         owner_repo = raw_url_base.split("github.com/")[-1]
         tag = pkg_info.get("tag")
         xml_name = pkg_info.get("package_xml_name", "package.xml")
-        raw_url = f"https://raw.githubusercontent.com/{owner_repo}/{tag}/{xml_name}"
+        additional_folder = pkg_info.get("additional_folder", "")
+        if additional_folder != "":
+            additional_folder = additional_folder + "/"
+        raw_url = f"https://raw.githubusercontent.com/{owner_repo}/{tag}/{additional_folder}{xml_name}"
         try:
             with urllib.request.urlopen(raw_url) as resp:
                 return resp.read().decode('utf-8')

vinca/main.py

@@ -1084,19 +1084,8 @@ def main():
 
     snapshot, additional_packages_snapshot = read_snapshot(vinca_conf)
 
-    from .template import generate_bld_ament_cmake
-    from .template import generate_bld_ament_python
-    from .template import generate_bld_catkin
-    from .template import generate_activate_hook
-    from .template import generate_bld_colcon_merge
-    from .template import generate_bld_catkin_merge
-
-    generate_bld_ament_cmake()
-    generate_bld_ament_python()
-    generate_bld_catkin()
-    generate_bld_colcon_merge()
-    generate_bld_catkin_merge()
-    generate_activate_hook()
+    # Store additional_packages_snapshot in vinca_conf for template access
+    vinca_conf["_additional_packages_snapshot"] = additional_packages_snapshot or {}
 
     if arguments.trigger_new_versions:
         vinca_conf["trigger_new_versions"] = True

vinca/template.py

@@ -130,24 +130,44 @@ def write_recipe(source, outputs, vinca_conf, single_file=True):
                     shutil.copyfile(p, recipe_dir / p)
 
             build_scripts = re.findall(r"'(.*?)'", meta["build"]["script"])
-            baffer = meta["build"]["script"]
             for script in build_scripts:
                 script_filename = script.replace("$RECIPE_DIR", "").replace("%RECIPE_DIR%", "").replace("/", "").replace("\\", "")
-                copyfile_with_exec_permissions(script_filename, recipe_dir / script_filename)
+                # Generate the build script directly in the recipe directory
+                # Get additional CMake arguments from pkg_additional_info
+                from vinca.utils import get_pkg_additional_info, ensure_name_is_without_distro_prefix_and_with_underscores
+                pkg_name = o["package"]["name"]
+                # Use the proper utility function to normalize the package name
+                pkg_shortname = ensure_name_is_without_distro_prefix_and_with_underscores(pkg_name, vinca_conf)
+
+                additional_cmake_args = ""
+                additional_folder = ""
+                if pkg_shortname:
+                    pkg_additional_info = get_pkg_additional_info(pkg_shortname, vinca_conf)
+                    additional_cmake_args = pkg_additional_info.get("additional_cmake_args", "")
+
+                    # Check if this package has folder info from additional_packages_snapshot
+                    if (vinca_conf.get("_additional_packages_snapshot") and
+                        pkg_shortname in vinca_conf["_additional_packages_snapshot"]):
+                        additional_folder = vinca_conf["_additional_packages_snapshot"][pkg_shortname].get("additional_folder", "")
+
+                generate_build_script_for_recipe(script_filename, recipe_dir / script_filename, additional_cmake_args, additional_folder)
             if "catkin" in o["package"]["name"] or "workspace" in o["package"]["name"]:
-                shutil.copyfile("activate.sh", recipe_dir / "activate.sh")
-                shutil.copyfile("activate.bat", recipe_dir / "activate.bat")
-                shutil.copyfile("activate.ps1", recipe_dir / "activate.ps1")
-                shutil.copyfile("deactivate.sh", recipe_dir / "deactivate.sh")
-                shutil.copyfile("deactivate.bat", recipe_dir / "deactivate.bat")
-                shutil.copyfile("deactivate.ps1", recipe_dir / "deactivate.ps1")
+                # Generate activation scripts directly in the recipe directory
+                generate_activation_scripts_for_recipe(recipe_dir)
 
-
-def generate_template(template_in, template_out):
+def generate_template(template_in, template_out, extra_globals=None):
     import em
     from vinca.config import skip_testing, ros_distro
 
-    g = {"ros_distro": ros_distro, "skip_testing": "ON" if skip_testing else "OFF"}
+    g = {
+        "ros_distro": ros_distro,
+        "skip_testing": "ON" if skip_testing else "OFF"
+    }
+
+    # Merge additional global variables if provided
+    if extra_globals:
+        g.update(extra_globals)
+
     interpreter = em.Interpreter(
         output=template_out, options={em.RAW_OPT: True, em.BUFFERED_OPT: True}
     )
@@ -163,81 +183,63 @@ def generate_template(template_in, template_out):
         # Set executable permissions for user, group, and others
         os.chmod(template_out.name, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
-def generate_bld_ament_cmake():
-    import pkg_resources
-
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/bld_ament_cmake.bat.in"
-    )
-    generate_template(template_in, open("bld_ament_cmake.bat", "w"))
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/build_ament_cmake.sh.in"
-    )
-    generate_template(template_in, open("build_ament_cmake.sh", "w"))
-
-
-def generate_bld_ament_python():
-    import pkg_resources
-
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/bld_ament_python.bat.in"
-    )
-    generate_template(template_in, open("bld_ament_python.bat", "w"))
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/build_ament_python.sh.in"
-    )
-    generate_template(template_in, open("build_ament_python.sh", "w"))
-
-
-def generate_bld_catkin():
-    import pkg_resources
-
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/bld_catkin.bat.in"
-    )
-    generate_template(template_in, open("bld_catkin.bat", "w"))
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/build_catkin.sh.in"
-    )
-    generate_template(template_in, open("build_catkin.sh", "w"))
-
-
-def generate_bld_colcon_merge():
-    import pkg_resources
-
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/bld_colcon_merge.bat.in"
-    )
-    generate_template(template_in, open("bld_colcon_merge.bat", "w"))
-
-
-def generate_bld_catkin_merge():
+def generate_build_script_for_recipe(script_name, output_path, additional_cmake_args="", additional_folder=""):
+    """Generate a specific build script directly in the recipe directory."""
     import pkg_resources
 
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/bld_catkin_merge.bat.in"
-    )
-    generate_template(template_in, open("bld_catkin_merge.bat", "w"))
+    # Map script names to their template files
+    script_templates = {
+        "build_ament_cmake.sh": "templates/build_ament_cmake.sh.in",
+        "bld_ament_cmake.bat": "templates/bld_ament_cmake.bat.in",
+        "build_ament_python.sh": "templates/build_ament_python.sh.in",
+        "bld_ament_python.bat": "templates/bld_ament_python.bat.in",
+        "build_catkin.sh": "templates/build_catkin.sh.in",
+        "bld_catkin.bat": "templates/bld_catkin.bat.in",
+        "bld_colcon_merge.bat": "templates/bld_colcon_merge.bat.in",
+        "bld_catkin_merge.bat": "templates/bld_catkin_merge.bat.in"
+    }
+
+    if script_name in script_templates:
+        template_in = pkg_resources.resource_filename("vinca", script_templates[script_name])
+        with open(output_path, "w") as output_file:
+            extra_globals = {}
+            if additional_cmake_args:
+                extra_globals["additional_cmake_args"] = additional_cmake_args
+            else:
+                extra_globals["additional_cmake_args"] = ""
+            if additional_folder:
+                extra_globals["additional_folder"] = additional_folder
+            else:
+                extra_globals["additional_folder"] = ""
+            generate_template(template_in, output_file, extra_globals)
 
+        # Set executable permissions on Unix systems
+        if os.name == 'posix' and script_name.endswith('.sh'):
+            current_permissions = os.stat(output_path).st_mode
+            os.chmod(output_path, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    else:
+        print(f"Warning: Unknown build script template for {script_name}")
 
-def generate_activate_hook():
+def generate_activation_scripts_for_recipe(recipe_dir):
+    """Generate activation scripts directly in the recipe directory."""
     import pkg_resources
 
-    template_in = pkg_resources.resource_filename("vinca", "templates/activate.bat.in")
-    generate_template(template_in, open("activate.bat", "w"))
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/deactivate.bat.in"
-    )
-    generate_template(template_in, open("deactivate.bat", "w"))
-
-    template_in = pkg_resources.resource_filename("vinca", "templates/activate.ps1.in")
-    generate_template(template_in, open("activate.ps1", "w"))
-    template_in = pkg_resources.resource_filename(
-        "vinca", "templates/deactivate.ps1.in"
-    )
-    generate_template(template_in, open("deactivate.ps1", "w"))
-
-    template_in = pkg_resources.resource_filename("vinca", "templates/activate.sh.in")
-    generate_template(template_in, open("activate.sh", "w"))
-    template_in = pkg_resources.resource_filename("vinca", "templates/deactivate.sh.in")
-    generate_template(template_in, open("deactivate.sh", "w"))
+    activation_templates = {
+        "activate.sh": "templates/activate.sh.in",
+        "activate.bat": "templates/activate.bat.in",
+        "activate.ps1": "templates/activate.ps1.in",
+        "deactivate.sh": "templates/deactivate.sh.in",
+        "deactivate.bat": "templates/deactivate.bat.in",
+        "deactivate.ps1": "templates/deactivate.ps1.in"
+    }
+
+    for script_name, template_path in activation_templates.items():
+        template_in = pkg_resources.resource_filename("vinca", template_path)
+        output_path = recipe_dir / script_name
+        with open(output_path, "w") as output_file:
+            generate_template(template_in, output_file)  # No extra globals needed for activation scripts
+
+        # Set executable permissions on Unix systems for shell scripts
+        if os.name == 'posix' and script_name.endswith('.sh'):
+            current_permissions = os.stat(output_path).st_mode
+            os.chmod(output_path, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

vinca/templates/bld_ament_cmake.bat.in

@@ -43,7 +43,8 @@ cmake ^
     -DCMAKE_OBJECT_PATH_MAX=255 ^
     --compile-no-warning-as-error ^
     -DPYTHON_INSTALL_DIR=%PYTHON_INSTALL_DIR% ^
-    %SRC_DIR%\%PKG_NAME%\src\work
+    @(additional_cmake_args) ^
+    %SRC_DIR%\%PKG_NAME%\src\work\@(additional_folder)
 if errorlevel 1 exit 1
 
 :: We explicitly pass %CPU_COUNT% to cmake --build as we are not using Ninja,

vinca/templates/bld_ament_python.bat.in

@@ -4,7 +4,7 @@ setlocal
 
 set "PYTHONPATH=%LIBRARY_PREFIX%\lib\site-packages;%SP_DIR%"
 
-pushd %SRC_DIR%\%PKG_NAME%\src\work
+pushd %SRC_DIR%\%PKG_NAME%\src\work\@(additional_folder)
 set "PKG_NAME_SHORT=%PKG_NAME:*ros-@(ros_distro)-=%"
 set "PKG_NAME_SHORT=%PKG_NAME_SHORT:-=_%"
 

vinca/templates/bld_catkin.bat.in

@@ -42,7 +42,8 @@ cmake ^
     -DBoost_USE_STATIC_LIBS=OFF ^
     %CATKIN_BUILD_BINARY_PACKAGE_ARGS% ^
     -DCATKIN_SKIP_TESTING=%SKIP_TESTING% ^
-    %SRC_DIR%\%PKG_NAME%\src\work
+    @(additional_cmake_args) ^
+    %SRC_DIR%\%PKG_NAME%\src\work\@(additional_folder)
 if errorlevel 1 exit 1
 
 if "%PKG_NAME%" == "ros-@(ros_distro)-eigenpy" (

vinca/templates/build_ament_cmake.sh.in

@@ -103,7 +103,7 @@ elif [ "${PKG_NAME}" == "ros-humble-wasm-cpp" ]; then
 elif [ "${PKG_NAME}" == "dynmsg" ]; then
   WORK_DIR=$SRC_DIR/$PKG_NAME/src/work/dynmsg
 else
-  WORK_DIR=$SRC_DIR/$PKG_NAME/src/work
+  WORK_DIR=$SRC_DIR/$PKG_NAME/src/work/@(additional_folder)
 fi;
 
 $CMAKE_GEN \
@@ -128,6 +128,7 @@ $CMAKE_GEN \
     -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \
     --compile-no-warning-as-error \
     $EXTRA_CMAKE_ARGS \
+    @(additional_cmake_args) \
     $WORK_DIR
 
 $CMAKE_BLD --build . --config $BUILD_TYPE --target install

vinca/templates/build_ament_python.sh.in

@@ -3,7 +3,7 @@
 
 set -eo pipefail
 
-pushd $SRC_DIR/$PKG_NAME/src/work
+pushd $SRC_DIR/$PKG_NAME/src/work/@(additional_folder)
 
 # If there is a setup.cfg that contains install-scripts then we should not set it here
 if [ -f setup.cfg ] && grep -q "install[-_]scripts" setup.cfg; then

vinca/templates/build_catkin.sh.in

@@ -111,8 +111,9 @@ cmake ${CMAKE_ARGS} --compile-no-warning-as-error \
          -DCATKIN_BUILD_BINARY_PACKAGE=$CATKIN_BUILD_BINARY_PACKAGE \
          -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \
          $EXTRA_CMAKE_ARGS \
+         @(additional_cmake_args) \
          -G "$GENERATOR" \
-         $SRC_DIR/$PKG_NAME/src/work
+         $SRC_DIR/$PKG_NAME/src/work/@(additional_folder)
 
 cmake --build . --config Release --target all