Merge branch 'main' of github.com:forcedotcom/datacloud-customcode-python-sdk into sarasavilli/func_pyfile_lib_support

Author: sarasavilli-sf

README.md

@@ -40,6 +40,8 @@ pip install salesforce-data-customcode
 datacustomcode init my_package
 ```
 
+To create a package of type function, pass the parameter `--code-type=function` with the init command.
+
 This will yield all necessary files to get started:
 ```
 .
@@ -244,6 +246,8 @@ Initialize a new development environment with a code package template.
 
 Argument:
 - `DIRECTORY`: Directory to create project in (default: ".")
+Options:
+- `--code-type TEXT`: This can be either `function` or `script`. The default value is `script` if the argument is missing.
 
 
 #### `datacustomcode scan`
@@ -293,6 +297,7 @@ Options:
 - `--description TEXT`: Description of the transformation job (default: "")
 - `--network TEXT`: docker network (default: "default")
 - `--cpu-size TEXT`: CPU size for the deployment (default: `CPU_2XL`). Available options: CPU_L(Large), CPU_XL(Extra Large), CPU_2XL(2X Large), CPU_4XL(4X Large)
+- `--function-invoke-opt TEXT`: Currently we support only `UnstructuredChunking` for functions.
 
 
 ## Docker usage

src/datacustomcode/cmd.py

@@ -85,9 +85,13 @@ def _cmd_output(
     **kwargs: Any,
 ) -> tuple[int, bytes, Union[bytes, None]]:
     _setdefault_kwargs(kwargs)
+    kwargs.setdefault("shell", True)
+    # On Windows, Popen with shell=True and a sequence uses list2cmdline() which
+    # quotes the entire string, causing cmd.exe to fail. Joining to a plain string
+    # works correctly on both Unix (/bin/sh -c "...") and Windows (cmd.exe /c ...).
+    cmd_arg: Union[tuple[str, ...], str] = " ".join(cmd) if kwargs.get("shell") else cmd
     try:
-        kwargs.setdefault("shell", True)
-        proc = subprocess.Popen(cmd, **kwargs)
+        proc = subprocess.Popen(cmd_arg, **kwargs)
     except OSError as e:
         returncode, stdout_b, stderr_b = _oserror_to_output(e)
     else:

src/datacustomcode/deploy.py

@@ -272,7 +272,7 @@ def create_deployment(
         raise
 
 
-PLATFORM_ENV_VAR = "DOCKER_DEFAULT_PLATFORM=linux/amd64"
+PLATFORM_ENV = {"DOCKER_DEFAULT_PLATFORM": "linux/amd64"}
 DOCKER_IMAGE_NAME = "datacloud-custom-code-dependency-builder"
 DEPENDENCIES_ARCHIVE_NAME = "native_dependencies"
 DEPENDENCIES_ARCHIVE_FULL_NAME = f"{DEPENDENCIES_ARCHIVE_NAME}.tar.gz"
@@ -289,18 +289,25 @@ def prepare_dependency_archive(
     cmd = f"docker images -q {DOCKER_IMAGE_NAME}"
     image_exists = cmd_output(cmd)
 
+    docker_env = {**os.environ, **PLATFORM_ENV}
+
     if not image_exists:
         logger.info(f"Building docker image with docker network: {docker_network}...")
         cmd = docker_build_cmd(docker_network)
-        cmd_output(cmd)
+        cmd_output(cmd, env=docker_env)
 
-    with tempfile.TemporaryDirectory() as temp_dir:
-        logger.info(f"Building dependencies with docker network: {docker_network}")
+    # ignore_cleanup_errors=True: on Windows, Docker creates files inside the
+    # mounted volume whose permissions prevent the host from deleting them.
+    # The archive has already been copied out, so silently skipping leftover
+    # files is safe and avoids a fatal error on context-manager exit.
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
+        logger.info(
+            f"Building dependencies archive with docker network: {docker_network}"
+        )
         shutil.copy("requirements.txt", temp_dir)
         shutil.copy("build_native_dependencies.sh", temp_dir)
         cmd = docker_run_cmd(docker_network, temp_dir)
-        cmd_output(cmd)
-
+        cmd_output(cmd, env=docker_env)
         if package_type == "function":
             source_py_files = os.path.join(temp_dir, "py-files")
             if os.path.exists(source_py_files):
@@ -326,23 +333,19 @@ def prepare_dependency_archive(
 
 
 def docker_build_cmd(network: str) -> str:
-    cmd = (
-        f"{PLATFORM_ENV_VAR} docker build -t {DOCKER_IMAGE_NAME} "
-        f"--file Dockerfile.dependencies . "
-    )
+    cmd = f"docker build -t {DOCKER_IMAGE_NAME} --file Dockerfile.dependencies . "
 
     if network != "default":
         cmd = cmd + f"--network {network}"
     logger.debug(f"Docker build command: {cmd}")
     return cmd
 
 
-def docker_run_cmd(network: str, temp_dir) -> str:
-    cmd = (
-        f"{PLATFORM_ENV_VAR} docker run --rm "
-        f"-v {temp_dir}:/workspace "
-        f"{DOCKER_IMAGE_NAME} "
-    )
+def docker_run_cmd(network: str, temp_dir: str) -> str:
+    # Normalise path separators: Docker expects forward slashes even on Windows,
+    # and quoting handles paths that contain spaces.
+    docker_path = temp_dir.replace("\\", "/")
+    cmd = f'docker run --rm -v "{docker_path}:/workspace" {DOCKER_IMAGE_NAME} '
 
     if network != "default":
         cmd = cmd + f"--network {network} "

src/datacustomcode/run.py

@@ -25,7 +25,7 @@
 )
 
 from datacustomcode.config import config
-from datacustomcode.scan import get_package_type
+from datacustomcode.scan import find_base_directory, get_package_type
 
 
 def _set_config_option(config_obj, key: str, value: str) -> None:
@@ -68,7 +68,9 @@ def run_entrypoint(
             f"config.json not found at {config_json_path}. config.json is required."
         )
 
-    package_type = get_package_type(entrypoint_dir)
+    # SDK config (.datacustomcode_proj/sdk_config.json) is under project root
+    base_directory = find_base_directory(entrypoint)
+    package_type = get_package_type(base_directory)
 
     try:
         with open(config_json_path, "r") as f:

src/datacustomcode/templates/function/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/emr-on-eks/spark/emr-7.3.0:latest
+FROM python:3.11-slim
 
 USER root