Function Support for pyfiles

Author: sarasavilli-sf

src/datacustomcode/deploy.py

@@ -279,10 +279,11 @@ def create_deployment(
 DEPENDENCIES_ARCHIVE_PATH = os.path.join(
     "payload", "archives", DEPENDENCIES_ARCHIVE_FULL_NAME
 )
+PY_FILES_PATH = os.path.join("payload", "py-files")
 ZIP_FILE_NAME = "deployment.zip"
 
 
-def prepare_dependency_archive(directory: str, docker_network: str) -> None:
+def prepare_dependency_archive(directory: str, docker_network: str, package_type: str) -> None:
     cmd = f"docker images -q {DOCKER_IMAGE_NAME}"
     image_exists = cmd_output(cmd)
 
@@ -299,11 +300,21 @@ def prepare_dependency_archive(directory: str, docker_network: str) -> None:
         shutil.copy("build_native_dependencies.sh", temp_dir)
         cmd = docker_run_cmd(docker_network, temp_dir)
         cmd_output(cmd)
-        archives_temp_path = os.path.join(temp_dir, DEPENDENCIES_ARCHIVE_FULL_NAME)
-        os.makedirs(os.path.dirname(DEPENDENCIES_ARCHIVE_PATH), exist_ok=True)
-        shutil.copy(archives_temp_path, DEPENDENCIES_ARCHIVE_PATH)
 
-        logger.info(f"Dependencies archived to {DEPENDENCIES_ARCHIVE_PATH}")
+        if package_type == "function":
+            source_py_files = os.path.join(temp_dir, "py-files")
+            os.makedirs(os.path.dirname(PY_FILES_PATH), exist_ok=True)
+            if os.path.exists(PY_FILES_PATH):
+                shutil.rmtree(PY_FILES_PATH)
+            shutil.copytree(source_py_files, PY_FILES_PATH)
+            logger.info(f"Dependencies copied to {PY_FILES_PATH}")
+        else:
+            archives_temp_path = os.path.join(
+                temp_dir, DEPENDENCIES_ARCHIVE_FULL_NAME
+            )
+            os.makedirs(os.path.dirname(DEPENDENCIES_ARCHIVE_PATH), exist_ok=True)
+            shutil.copy(archives_temp_path, DEPENDENCIES_ARCHIVE_PATH)
+            logger.info(f"Dependencies archived to {DEPENDENCIES_ARCHIVE_PATH}")
 
 
 def docker_build_cmd(network: str) -> str:
@@ -511,16 +522,28 @@ def upload_zip(file_upload_url: str) -> None:
         response.raise_for_status()
 
 
+def _get_package_type_for_directory(directory: str) -> str:
+    """Resolve package type (script/function) for the given payload directory."""
+    try:
+        entrypoint_path = os.path.join(os.path.abspath(directory), "entrypoint.py")
+        base_directory = find_base_directory(entrypoint_path)
+        return get_package_type(base_directory)
+    except (ValueError, FileNotFoundError):
+        return "script"
+
+
 def zip(
     directory: str,
     docker_network: str,
 ):
     # Create a zip file excluding .DS_Store files
     import zipfile
 
+    package_type = _get_package_type_for_directory(directory)
+
     # prepare payload only if requirements.txt is non-empty
     if has_nonempty_requirements_file(directory):
-        prepare_dependency_archive(directory, docker_network)
+        prepare_dependency_archive(directory, docker_network, package_type)
     else:
         logger.info(
             f"Skipping dependency archive: requirements.txt is missing or empty "

src/datacustomcode/templates/function/build_native_dependencies.sh

@@ -1,9 +1,7 @@
 #!/bin/bash
 set -e
 
-# Description: build native dependencies
+# Description: build native dependencies for function (unpacked pip install to py-files)
 
 python3.11 -m venv --copies .venv
-source .venv/bin/activate
-pip install -r requirements.txt
-venv-pack -o native_dependencies.tar.gz -f
+source .venv/bin/activate

src/datacustomcode/templates/function/requirements.txt

@@ -1 +1 @@
-# Packages required for the custom code
+# Packages required for the custom code

tests/test_deploy.py

@@ -611,7 +611,7 @@ def test_zip_with_requirements(
         zip("/test/dir", "default")
 
         mock_has_requirements.assert_called_once_with("/test/dir")
-        mock_prepare.assert_called_once_with("/test/dir", "default")
+        mock_prepare.assert_called_once_with("/test/dir", "default", "script")
         mock_zipfile.assert_called_once_with(
             "deployment.zip", "w", zipfile.ZIP_DEFLATED
         )

tests/test_scan.py

@@ -8,9 +8,11 @@
 import pytest
 
 from datacustomcode.scan import (
-    SDK_CONFIG_DIR,
     DataAccessLayerCalls,
+    SDK_CONFIG_DIR,
+    SDK_CONFIG_FILE,
     dc_config_json_from_file,
+    get_sdk_config_path,
     scan_file,
     scan_file_for_imports,
     update_config,
@@ -40,7 +42,7 @@ def create_sdk_config(base_directory: str, package_type: str = "script") -> str:
     """
     sdk_config = {"type": package_type}
     write_sdk_config(base_directory, sdk_config)
-    return os.path.join(base_directory, SDK_CONFIG_DIR, "config.json")
+    return os.path.join(base_directory, SDK_CONFIG_DIR, SDK_CONFIG_FILE)
 
 
 class TestClientMethodVisitor: