diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 798e431..72e2796 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} pytest-args: >- - -v -ra -W error + -v -ra -W error -m "not package_manager_install" -W ignore::DeprecationWarning:jupyter_client.session -W ignore::DeprecationWarning:orangecanvas.utils.localization -W ignore::DeprecationWarning:orangecanvas.utils.localization.si @@ -79,6 +79,25 @@ jobs: enable-coverage: "false" extra-pytest-warnings: "" + # Run the tests that install package managers, on runners without them + tests-package-manager-install: + needs: build + uses: ewoks-kit/.github/.github/workflows/python-tests.yml@main + with: + os: ${{ matrix.os }} + python-version: "3.12" + pytest-args: >- + -v -ra -W error -m package_manager_install + -W ignore::DeprecationWarning:jupyter_client.session + -W ignore::DeprecationWarning:orangecanvas.utils.localization + -W ignore::DeprecationWarning:orangecanvas.utils.localization.si + -W ignore::DeprecationWarning:matplotlib._fontconfig_pattern + -W ignore::DeprecationWarning:matplotlib._mathtext + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + # Run linter / checks checks: uses: ewoks-kit/.github/.github/workflows/python-check.yml@main diff --git a/doc/conf.py b/doc/conf.py index fcef75b..6748682 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -3,6 +3,7 @@ import importlib.metadata +import os release = importlib.metadata.version("ewoks") @@ -48,6 +49,7 @@ html_title = docstitle html_logo = "_static/logo.png" html_static_path = ["_static"] +html_extra_path = ["../src/ewoks/_bootstrap"] html_template_path = ["_templates"] html_css_files = ["custom.css"] @@ -76,3 +78,20 @@ "footer_start": ["copyright"], "footer_end": ["footer_end"], } + +# Root of the documentation version that is built: on Read the Docs, for example +# .../en/stable, or on GitLab Pages +_DOCS_URL = ( + os.environ.get("READTHEDOCS_CANONICAL_URL") + or os.environ.get("CI_PAGES_URL") + or "https://ewoks.readthedocs.io/en/latest" +).rstrip("/") + + +def _substitute_docs_url(app, docname, source): + # Substitutions are not supported in code blocks + source[0] = source[0].replace("|docs_url|", _DOCS_URL) + + +def setup(app): + app.connect("source-read", _substitute_docs_url) diff --git a/doc/explanations/task_input_priority.rst b/doc/explanations/task_input_priority.rst index f003e44..d08274e 100644 --- a/doc/explanations/task_input_priority.rst +++ b/doc/explanations/task_input_priority.rst @@ -7,7 +7,7 @@ Details A :term:`node ` in a :term:`workflow` can get inputs from three different sources: 1. Via the ``data_mapping`` :term:`link ` attribute of an incoming :term:`link ` (see `Link attributes `_) -2. Via the ``parameters`` CLI argument (or ``inputs`` for Python) when executing/submitting the :term:`workflow` (see `ewoks execute reference `_) +2. Via the ``parameters`` CLI argument (or ``inputs`` for Python) when executing/submitting the :term:`workflow` (see :ref:`ewoks execute reference `) 3. Via the ``default_inputs`` :term:`node ` attribute of the :term:`node ` itself (see `Node attributes `_) If the same input is specified by these different sources, :term:`Ewoks` applies the following priorities: diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst index 74867f6..58c7c7f 100644 --- a/doc/howtoguides/requirements.rst +++ b/doc/howtoguides/requirements.rst @@ -13,6 +13,9 @@ What is stored ``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store +* ``ewoks``: the version of :term:`ewoks` that generated the requirements and the engine + that saved the :term:`workflow`. The + :ref:`ewoks-install script ` installs both to run ``ewoks install``. * ``python`` and ``system``: the python interpreter and the operating system. * ``distributions``: every installed python package with its version and, when it was not installed from the python package index, the git commit or the archive it came from. Any diff --git a/doc/reference/cli.rst b/doc/reference/cli.rst index 4add2cd..a68fe21 100644 --- a/doc/reference/cli.rst +++ b/doc/reference/cli.rst @@ -63,6 +63,8 @@ ewoks convert **ewoks convert** can also be used to store ``inputs`` inside the destination :term:`workflow`. +.. _cli_execute: + ewoks execute ------------- diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index 96dd6f7..64b1f25 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -40,6 +40,7 @@ Store the requirements .. code-block:: json { + "ewoks": {"version": "7.0.0", "engine": {"name": "ewokscore", "version": "5.1.0"}}, "python": {"version": "3.12.11", "implementation": "CPython", "...": "..."}, "system": {"system": "Linux", "machine": "x86_64", "...": "..."}, "distributions": [ @@ -53,6 +54,9 @@ Store the requirements } } +* ``ewoks`` is the version of :term:`ewoks` that generated the requirements and the engine that + saved the :term:`workflow`: ``ewokscore`` for a ``.json`` or ``.yaml`` file and + ``ewoksorange`` for an ``.ows`` file. * ``distributions`` are the installed python packages. Any :term:`package manager` can recreate the environment from this list. * ``manager`` is the :term:`package manager` that generated the requirements together with @@ -63,19 +67,67 @@ Create the environment ``ewoks install`` creates a python environment for the :term:`workflow`. It prints the python interpreter of the environment it created, the command to execute the :term:`workflow` in it -and the command to remove it again +and the command to remove it again. These commands work from any terminal and directory .. code-block:: text Installed requirements for demo.json - Python : ewoks_envs/demo/bin/python - Execute: ewoks execute --env ewoks_envs/demo demo.json - Remove : rm -rf ewoks_envs/demo + Python : /home/user/ewoks_envs/demo/bin/python + Execute: /home/user/ewoks_envs/demo/bin/python -m ewoks execute /home/user/demo.json + Remove : rm -rf /home/user/ewoks_envs/demo Without ``--yes`` you are asked to confirm after the packages have been listed. The walk-throughs use ``--env-root`` to create the environment in the working directory. Without it the environment is created where the :term:`package manager` creates named environments. +.. _install_bootstrap: + +Create the environment without ewoks +++++++++++++++++++++++++++++++++++++ + +``ewoks install`` needs a python environment with :term:`ewoks`. The ``ewoks-install`` script +only needs a :term:`package manager`: it installs the version of :term:`ewoks` that generated +the requirements, and the engine that saved the :term:`workflow`, in a bootstrap environment +and runs ``ewoks install`` from there. All +arguments are passed to ``ewoks install``, except for the options of the script itself + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json --package-manager-name uv + +* ``--package-manager-name`` and ``--package-manager-command`` select the + :term:`package manager` that creates the bootstrap environment. Without them it is the first + one that is available of uv, pixi, conda, poetry and pip-venv. +* ``--print-command`` prints the ``ewoks install`` command instead of running it. +* ``--ewoks-requirement`` installs another version of :term:`ewoks`, for example + ``--ewoks-requirement "ewoks>=7"``. Repeat it to install an engine as well, for example + ``--ewoks-requirement ewoks --ewoks-requirement ewoksmyengine``. The latest version of + :term:`ewoks` is installed when the requirements do not provide one. +* ``--bootstrap-dir`` is the directory of the bootstrap environments. +* ``--install-package-manager`` installs the :term:`package manager` of + ``--package-manager-name`` in the bootstrap directory when it is not installed. + +The script keeps the bootstrap environment in ``~/.ewoks/bootstrap`` and uses it again when +it installs another :term:`workflow` whose requirements were generated by the same version of +:term:`ewoks`. The bootstrap environment only runs ``ewoks install``: +it is not the environment of the :term:`workflow`. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/conda.rst b/doc/tutorials/install/conda.rst index ee665bf..381cd1c 100644 --- a/doc/tutorials/install/conda.rst +++ b/doc/tutorials/install/conda.rst @@ -124,6 +124,33 @@ The environment of the :term:`workflow` is a conda environment in ``ewoks_envs/d ``--env-root`` it is created in the first environment directory of conda, where ``conda activate demo`` finds it. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name conda --yes --env-root ewoks_envs + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name conda --yes --env-root ewoks_envs + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json --package-manager-name conda --yes --env-root ewoks_envs + +The script does not install ``ewoks`` in the current environment, so execute the +workflow with the python interpreter of the environment of the :term:`workflow`, for example +``ewoks_envs/demo/bin/python -m ewoks execute demo.json``. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/pip_venv.rst b/doc/tutorials/install/pip_venv.rst index 2ee3857..aca58cb 100644 --- a/doc/tutorials/install/pip_venv.rst +++ b/doc/tutorials/install/pip_venv.rst @@ -83,6 +83,33 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a virtual environment in ``ewoks_envs/demo``. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name pip-venv --yes --env-root ewoks_envs + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name pip-venv --yes --env-root ewoks_envs + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json --package-manager-name pip-venv --yes --env-root ewoks_envs + +The script does not install ``ewoks`` in the current environment, so execute the +workflow with the python interpreter of the environment of the :term:`workflow`, for example +``ewoks_envs/demo/bin/python -m ewoks execute demo.json``. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/pixi.rst b/doc/tutorials/install/pixi.rst index 3a420dd..e1deec3 100644 --- a/doc/tutorials/install/pixi.rst +++ b/doc/tutorials/install/pixi.rst @@ -119,6 +119,33 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a pixi workspace in ``ewoks_envs/demo`` with the environment in its ``.pixi/envs/default`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name pixi --yes --env-root ewoks_envs + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name pixi --yes --env-root ewoks_envs + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json --package-manager-name pixi --yes --env-root ewoks_envs + +The script does not install ``ewoks`` in the current environment, so execute the +workflow with the python interpreter of the environment of the :term:`workflow`, for example +``ewoks_envs/demo/.pixi/envs/default/bin/python -m ewoks execute demo.json``. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/poetry.rst b/doc/tutorials/install/poetry.rst index f4360bd..c6175e0 100644 --- a/doc/tutorials/install/poetry.rst +++ b/doc/tutorials/install/poetry.rst @@ -106,6 +106,33 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a poetry project in ``ewoks_envs/demo`` with the virtual environment in its ``.venv`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name poetry --yes --env-root ewoks_envs + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name poetry --yes --env-root ewoks_envs + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json --package-manager-name poetry --yes --env-root ewoks_envs + +The script does not install ``ewoks`` in the current environment, so execute the +workflow with the python interpreter of the environment of the :term:`workflow`, for example +``ewoks_envs/demo/.venv/bin/python -m ewoks execute demo.json``. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/uv.rst b/doc/tutorials/install/uv.rst index 11db413..e98a5d6 100644 --- a/doc/tutorials/install/uv.rst +++ b/doc/tutorials/install/uv.rst @@ -113,6 +113,33 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a uv project in ``ewoks_envs/demo`` with the virtual environment in its ``.venv`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv --yes --env-root ewoks_envs + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv --yes --env-root ewoks_envs + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json --package-manager-name uv --yes --env-root ewoks_envs + +The script does not install ``ewoks`` in the current environment, so execute the +workflow with the python interpreter of the environment of the :term:`workflow`, for example +``ewoks_envs/demo/.venv/bin/python -m ewoks execute demo.json``. + Execute the workflow -------------------- diff --git a/pyproject.toml b/pyproject.toml index cebf15e..6b60671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ ewoks = "ewoks.__main__:main" [tool.setuptools.package-data] "ewoks.tests.notebooks" = ["*.ipynb"] +"ewoks._bootstrap" = ["*.sh", "*.ps1"] [tool.ruff.lint] select = [ diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 1383a56..29686c5 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -1,3 +1,5 @@ +import os +import shlex import subprocess import sys import traceback @@ -211,7 +213,16 @@ def command_install( location = environment.location print(f" Python : {environment.python}") if environment.distribution_version("ewoks"): - print(f" Execute: ewoks execute --env {location} {workflow}") + execute = _shell_command( + [ + str(environment.python), + "-m", + "ewoks", + "execute", + _absolute_workflow(workflow), + ] + ) + print(f" Execute: {execute}") else: print( " Execute: the environment has no ewoks to execute the " @@ -227,7 +238,21 @@ def _remove_command(location: Path) -> str: """Shell command that removes a python environment.""" if sys.platform == "win32": return f'rmdir /s /q "{location}"' - return f"rm -rf {location}" + return f"rm -rf {shlex.quote(str(location))}" + + +def _shell_command(arguments: List[str]) -> str: + """Command line for the shell of the platform.""" + if sys.platform == "win32": + return subprocess.list2cmdline(arguments) + return shlex.join(arguments) + + +def _absolute_workflow(workflow: str) -> str: + """Workflow that does not depend on the working directory.""" + if os.path.isfile(workflow): + return os.path.abspath(workflow) + return workflow def command_in_environment( diff --git a/src/ewoks/_bootstrap/ewoks-install.ps1 b/src/ewoks/_bootstrap/ewoks-install.ps1 new file mode 100644 index 0000000..3cfc9e7 --- /dev/null +++ b/src/ewoks/_bootstrap/ewoks-install.ps1 @@ -0,0 +1,520 @@ +# Install the python environment of an ewoks workflow with nothing but a package +# manager. For example +# +# & ([scriptblock]::Create((irm https://ewoks.readthedocs.io/en/stable/ewoks-install.ps1))) demo.json --package-manager-name uv +# +# Errors are thrown instead of calling `exit`, which would close the PowerShell +# session that runs the script block. + +$ErrorActionPreference = "Stop" + +$usage = @' +Usage: ewoks-install.ps1 [OPTIONS] WORKFLOW [ARGS ...] + +Install the python environment of an ewoks workflow with nothing but a package +manager. The ewoks version that generated the workflow requirements, and the +engine that saved the workflow, are installed in a bootstrap environment, which +then runs + + ewoks install WORKFLOW ARGS + +See 'ewoks install --help' for ARGS. The package manager that creates the +bootstrap environment is selected with '--package-manager-name' and +'--package-manager-command', which are passed to 'ewoks install' as well. +Without them it is the first available of uv, pixi, conda, poetry and pip-venv. +A package manager that is not installed can be installed in the bootstrap +directory with '--install-package-manager'. + +Options: + --print-command Print the 'ewoks install' command instead of + running it. + --ewoks-requirement REQ Install REQ instead of the ewoks version and engine + of the workflow, for example 'ewoks>=7' or a wheel + file. Repeat it to install more, for example + '--ewoks-requirement ewoks --ewoks-requirement + ewoksmyengine'. + --bootstrap-dir DIR Directory of the bootstrap environments. + Default: ~/.ewoks/bootstrap + --install-package-manager + Install the package manager of + '--package-manager-name' when it is not installed. + -h, --help Show this help. +'@ + +$onWindows = ($PSVersionTable.PSEdition -eq "Desktop") -or $IsWindows + +function Write-Info([string]$message) { + [Console]::Error.WriteLine("ewoks-install: $message") +} + +# Run a native command with its output on stderr instead of in the pipeline +function Invoke-Native([string[]]$command) { + $arguments = @() + if ($command.Count -gt 1) { + $arguments = $command[1..($command.Count - 1)] + } + & $command[0] @arguments 2>&1 | ForEach-Object { [Console]::Error.WriteLine($_) } + if ($LASTEXITCODE -ne 0) { + throw "command failed: $($command -join ' ')" + } +} + +function Test-Command([string]$name) { + return [bool](Get-Command $name -CommandType Application -ErrorAction SilentlyContinue) +} + +# Python interpreter with this major.minor version, or the default one +function Find-Python([string]$majorMinor) { + if ($onWindows) { + if (Test-Command "py") { + if ($majorMinor) { + & py "-$majorMinor" -c "pass" 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + return @("py", "-$majorMinor") + } + } + return @("py", "-3") + } + } elseif ($majorMinor -and (Test-Command "python$majorMinor")) { + return @("python$majorMinor") + } elseif (Test-Command "python3") { + return @("python3") + } + if (Test-Command "python") { + return @("python") + } + throw "no python interpreter found" +} + +function Get-PythonMajorMinor([string[]]$python) { + $arguments = @() + if ($python.Count -gt 1) { + $arguments = $python[1..($python.Count - 1)] + } + $majorMinor = & $python[0] @arguments -c 'import sys; print(''%d.%d'' % sys.version_info[:2])' + if ($LASTEXITCODE -ne 0) { + throw "cannot run $($python -join ' ')" + } + return "$majorMinor".Trim() +} + +# Value in the requirements of a workflow file, for example `ewoks.version`. The +# requirements are JSON, which can be embedded in another format, or YAML. +function Get-RequirementValue([string]$content, [string]$key) { + $value = Get-JsonRequirementValue $content $key + if (-not $value) { + $value = Get-YamlRequirementValue $content $key + } + return $value +} + +function Get-JsonRequirementValue([string]$content, [string]$key) { + $q = "[""']" + $value = "$q\s*:\s*$q([^""']*)$q" + # The members of an object, which can contain objects themselves + $members = "[^{}]*(?:\{[^{}]*\}[^{}]*)*" + switch ($key) { + "ewoks.version" { $pattern = "${q}ewoks$q\s*:\s*\{$members${q}version$value" } + "ewoks.engine.name" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}name$value" } + "ewoks.engine.version" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}version$value" } + "python.version" { $pattern = "${q}python$q\s*:\s*\{[^{}]*${q}version$value" } + } + $found = [regex]::Matches($content.Replace(""", '"'), $pattern) + if ($found.Count -eq 0) { + return "" + } + return $found[$found.Count - 1].Groups[1].Value +} + +function Get-YamlRequirementValue([string]$content, [string]$key) { + $wanted = "requirements.$key" + $indents = New-Object System.Collections.Generic.List[int] + $keys = New-Object System.Collections.Generic.List[string] + $found = "" + foreach ($line in ($content -split "\r?\n")) { + if ($line -match '^\s*(#|$)') { + continue + } + # The dash of a list item belongs to the indentation of its first key + $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_]+):\s*(.*)$') + if (-not $match.Success) { + continue + } + $indent = $match.Groups[1].Length + while ($indents.Count -gt 0 -and $indents[$indents.Count - 1] -ge $indent) { + $indents.RemoveAt($indents.Count - 1) + $keys.RemoveAt($keys.Count - 1) + } + $indents.Add($indent) + $keys.Add($match.Groups[2].Value) + $path = $keys -join "." + $value = $match.Groups[3].Value + if ($value -and (($path -eq $wanted) -or $path.EndsWith(".$wanted"))) { + $found = $value -replace "^[""']|[""']$", "" + } + } + return $found +} + +# Version of an installed distribution, or nothing when it is not installed +function Get-DistributionVersion([string]$python, [string]$name) { + try { + return "$(& $python -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' $name 2>$null)".Trim() + } catch { + return "" + } +} + +# Argument quoted for PowerShell when needed +function Format-Argument([string]$argument) { + if ($argument -match '^[A-Za-z0-9_./=:@%+,\\-]+$') { + return $argument + } + return "'" + $argument.Replace("'", "''") + "'" +} + +# Executable of a package manager installed in a directory by this script +function Get-ManagerExecutable([string]$manager, [string]$directory) { + $suffix = "" + if ($onWindows) { $suffix = ".exe" } + if ($manager -eq "conda") { + if ($onWindows) { return Join-Path (Join-Path $directory "Scripts") "conda.exe" } + return Join-Path (Join-Path $directory "bin") "conda" + } + return Join-Path (Join-Path $directory "bin") "$manager$suffix" +} + +# Run a native command with additional environment variables +function Invoke-NativeWithEnv([hashtable]$variables, [string[]]$command) { + $previous = @{} + foreach ($name in $variables.Keys) { + $previous[$name] = [Environment]::GetEnvironmentVariable($name) + [Environment]::SetEnvironmentVariable($name, $variables[$name]) + } + try { + Invoke-Native $command + } finally { + foreach ($name in $variables.Keys) { + [Environment]::SetEnvironmentVariable($name, $previous[$name]) + } + } +} + +# Install a package manager in a directory with its official installer +function Install-Manager([string]$manager, [string]$directory) { + Write-Info "install $manager in $directory" + if (Test-Path -LiteralPath $directory) { + Remove-Item -LiteralPath $directory -Recurse -Force + } + New-Item -ItemType Directory -Path (Split-Path -Parent $directory) -Force | Out-Null + # The PowerShell installers of uv and pixi only install Windows binaries + # and exit their PowerShell session + if ($onWindows) { + $suffix = ".ps1" + $runInstaller = @((Get-Process -Id $PID).Path, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File") + } else { + $suffix = ".sh" + $runInstaller = @("sh") + } + $temp = [System.IO.Path]::GetTempPath() + $installer = Join-Path $temp ("ewoks-install-" + [guid]::NewGuid()) + try { + switch ($manager) { + "uv" { + $installer += $suffix + Invoke-WebRequest -UseBasicParsing -Uri "https://astral.sh/uv/install$suffix" -OutFile $installer + Invoke-NativeWithEnv @{ UV_INSTALL_DIR = (Join-Path $directory "bin"); UV_NO_MODIFY_PATH = "1" } ($runInstaller + @($installer)) + } + "pixi" { + $installer += $suffix + Invoke-WebRequest -UseBasicParsing -Uri "https://pixi.sh/install$suffix" -OutFile $installer + Invoke-NativeWithEnv @{ PIXI_HOME = $directory; PIXI_NO_PATH_UPDATE = "1" } ($runInstaller + @($installer)) + } + "poetry" { + $installer += ".py" + Invoke-WebRequest -UseBasicParsing -Uri "https://install.python-poetry.org" -OutFile $installer + Invoke-NativeWithEnv @{ POETRY_HOME = $directory } (@(Find-Python "") + @($installer)) + } + "conda" { + $url = "https://github.com/conda-forge/miniforge/releases/latest/download" + if ($onWindows) { + $installer += ".exe" + Invoke-WebRequest -UseBasicParsing -Uri "$url/Miniforge3-Windows-x86_64.exe" -OutFile $installer + # '/D' must be the last argument and must not be quoted + Start-Process -Wait -FilePath $installer -ArgumentList "/S", "/D=$directory" + } else { + $installer += ".sh" + $system = "$(uname -s)".Trim() + if ($system -eq "Darwin") { $system = "MacOSX" } + $machine = "$(uname -m)".Trim() + Invoke-WebRequest -UseBasicParsing -Uri "$url/Miniforge3-$system-$machine.sh" -OutFile $installer + Invoke-Native @("bash", $installer, "-b", "-p", $directory) + } + } + } + } finally { + Remove-Item -LiteralPath $installer -Force -ErrorAction SilentlyContinue + } +} + +function Invoke-EwoksInstall([object[]]$arguments) { + # --- Arguments: the bootstrap options are removed, the others are kept --- + + $printCommand = $false + $installPackageManager = $false + $requirements = @() + $root = Join-Path (Join-Path $HOME ".ewoks") "bootstrap" + $manager = "" + $managerCommand = "" + $workflow = "" + $forwarded = @() + $i = 0 + while ($i -lt $arguments.Count) { + $arg = [string]$arguments[$i] + $i++ + if ($arg -in @("-h", "--help")) { + Write-Output $usage + return + } + if ($arg -eq "--print-command") { + $printCommand = $true + continue + } + if ($arg -eq "--install-package-manager") { + $installPackageManager = $true + continue + } + if ($arg -eq "--ewoks-requirement") { + if ($i -ge $arguments.Count) { throw "$arg requires a value" } + $requirements += [string]$arguments[$i] + $i++ + continue + } + if ($arg.StartsWith("--ewoks-requirement=")) { + $requirements += $arg.Substring($arg.IndexOf("=") + 1) + continue + } + if ($arg -eq "--bootstrap-dir") { + if ($i -ge $arguments.Count) { throw "$arg requires a value" } + $root = [string]$arguments[$i] + $i++ + continue + } + if ($arg.StartsWith("--bootstrap-dir=")) { + $root = $arg.Substring($arg.IndexOf("=") + 1) + continue + } + $forwarded += $arg + if ($arg -in @("--package-manager-name", "--package-manager-command")) { + if ($i -ge $arguments.Count) { throw "$arg requires a value" } + $value = [string]$arguments[$i] + $i++ + $forwarded += $value + if ($arg -eq "--package-manager-name") { + $manager = $value + } else { + $managerCommand = $value + } + } elseif ($arg.StartsWith("--package-manager-name=")) { + $manager = $arg.Substring($arg.IndexOf("=") + 1) + } elseif ($arg.StartsWith("--package-manager-command=")) { + $managerCommand = $arg.Substring($arg.IndexOf("=") + 1) + } elseif (-not $arg.StartsWith("-") -and -not $workflow -and (Test-Path -LiteralPath $arg -PathType Leaf)) { + $workflow = $arg + } + } + + if (-not $workflow) { + Write-Info $usage + throw "provide the file of the workflow" + } + + # --- Package manager --- + + $manager = $manager.ToLower() + if ($installPackageManager -and -not $manager) { + throw "--install-package-manager requires --package-manager-name" + } + if (-not $manager) { + if ($managerCommand) { + throw "--package-manager-command requires --package-manager-name" + } + foreach ($name in @("uv", "pixi", "conda", "poetry")) { + if (Test-Command $name) { + $manager = $name + break + } + } + if (-not $manager) { + $manager = "pip-venv" + } + } + + $cmd = @() + if ($managerCommand) { + $cmd = @($managerCommand -split '\s+' | Where-Object { $_ }) + } + if ($manager -in @("uv", "pixi", "conda", "poetry")) { + if (-not $cmd) { $cmd = @($manager) } + } elseif ($manager -ne "pip-venv") { + throw "package manager '$manager' is not supported" + } + + # `ewoks install` needs the package manager this script installed + if ($cmd -and -not $managerCommand -and -not (Test-Command $cmd[0])) { + $managerDir = Join-Path (Join-Path $root "package-managers") $manager + $executable = Get-ManagerExecutable $manager $managerDir + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + if (-not $installPackageManager) { + throw "$manager is not installed (see --install-package-manager)" + } + Install-Manager $manager $managerDir + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + throw "cannot install $manager in $managerDir" + } + } + $cmd = @($executable) + $forwarded += @("--package-manager-command", $executable) + } + + # --- Ewoks, engine and python versions of the workflow requirements --- + + $content = [System.IO.File]::ReadAllText((Resolve-Path -LiteralPath $workflow)) + $version = Get-RequirementValue $content "ewoks.version" + $engine = Get-RequirementValue $content "ewoks.engine.name" + $engineVersion = Get-RequirementValue $content "ewoks.engine.version" + $pythonVersion = Get-RequirementValue $content "python.version" + if (-not $engineVersion) { + $engine = "" + } + $pythonMajorMinor = "" + if ($pythonVersion -match '^(\d+\.\d+)') { + $pythonMajorMinor = $Matches[1] + } + + # The bootstrap environment of pinned requirements is up to date when it has their versions + $pinned = $false + if ($requirements) { + $sha = [System.Security.Cryptography.SHA256]::Create() + $hash = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($requirements -join " ")) + $tag = "custom-" + (($hash[0..4] | ForEach-Object { $_.ToString("x2") }) -join "") + } elseif ($version) { + $pinned = $true + $tag = $version + $requirements = @("ewoks==$version") + if ($engine) { + $tag = "$tag-$engine-$engineVersion" + $requirements += "$engine==$engineVersion" + } + } else { + Write-Info "the workflow does not provide the ewoks version: using the latest one" + $tag = "latest" + $requirements = @("ewoks>=7") + } + + # pip-venv and poetry cannot provide a python version: the interpreter decides + $basePython = @() + if ($manager -eq "pip-venv") { + if (-not $cmd) { $cmd = @(Find-Python $pythonMajorMinor) } + $pythonMajorMinor = Get-PythonMajorMinor $cmd + } elseif ($manager -eq "poetry") { + $basePython = @(Find-Python $pythonMajorMinor) + $pythonMajorMinor = Get-PythonMajorMinor $basePython + } + + $name = "ewoks-$tag" + if ($pythonMajorMinor) { + $name = "$name-python$pythonMajorMinor" + } + $location = Join-Path (Join-Path $root $manager) $name + + # --- Bootstrap environment --- + + if ($onWindows) { + $venvPython = Join-Path "Scripts" "python.exe" + $prefixPython = "python.exe" + } else { + $venvPython = Join-Path "bin" "python" + $prefixPython = $venvPython + } + switch ($manager) { + "pixi" { $python = Join-Path $location (Join-Path ".pixi/envs/default" $prefixPython) } + "conda" { $python = Join-Path $location $prefixPython } + "poetry" { $python = Join-Path $location (Join-Path ".venv" $venvPython) } + default { $python = Join-Path $location $venvPython } + } + + $pythonSpec = "python" + if ($pythonMajorMinor) { + $pythonSpec = "python=$pythonMajorMinor" + } + if (-not (Test-Path -LiteralPath $python -PathType Leaf)) { + Write-Info "create the bootstrap environment $location" + # A previous attempt failed + if (Test-Path -LiteralPath $location) { + Remove-Item -LiteralPath $location -Recurse -Force + } + New-Item -ItemType Directory -Path $location -Force | Out-Null + switch ($manager) { + "uv" { + $create = $cmd + @("venv", "--quiet") + if ($pythonMajorMinor) { $create += @("--python", $pythonMajorMinor) } + Invoke-Native ($create + @($location)) + } + "pixi" { + Invoke-Native ($cmd + @("init", $location)) + Invoke-Native ($cmd + @("add", "--manifest-path", (Join-Path $location "pixi.toml"), $pythonSpec, "pip")) + } + "conda" { + Invoke-Native ($cmd + @("create", "--yes", "--quiet", "--prefix", $location, $pythonSpec, "pip")) + } + "poetry" { + $pyproject = "[tool.poetry]`npackage-mode = false`n`n[tool.poetry.dependencies]`npython = `"*`"`n" + [System.IO.File]::WriteAllText((Join-Path $location "pyproject.toml"), $pyproject) + $interpreter = & $basePython[0] @($basePython | Select-Object -Skip 1) -c "import sys; print(sys.executable)" + # Poetry uses an active virtual or conda environment instead of the one of the project. + # Without its own python it runs the `python` command, which can be python 2. + $variables = @{ + VIRTUAL_ENV = $null + CONDA_PREFIX = $null + POETRY_VIRTUALENVS_IN_PROJECT = "true" + POETRY_VIRTUALENVS_USE_POETRY_PYTHON = "true" + } + Invoke-NativeWithEnv $variables ($cmd + @("-C", $location, "env", "use", "$interpreter".Trim())) + } + "pip-venv" { + Invoke-Native ($cmd + @("-m", "venv", $location)) + } + } + } + + $upToDate = $pinned -and ((Get-DistributionVersion $python "ewoks") -eq $version) + if ($upToDate -and $engine) { + $upToDate = (Get-DistributionVersion $python $engine) -eq $engineVersion + } + + if (-not $upToDate) { + Write-Info "install $($requirements -join ' ') in $location" + try { + if ($manager -eq "uv") { + Invoke-Native ($cmd + @("pip", "install", "--quiet", "--upgrade", "--python", $python) + $requirements) + } else { + Invoke-Native (@($python, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--upgrade") + $requirements) + } + } catch { + throw "cannot install $($requirements -join ' ') (see --ewoks-requirement)" + } + } + + # --- ewoks install --- + + $install = @("-m", "ewoks", "install") + $forwarded + if ($printCommand) { + $formatted = @($install | ForEach-Object { Format-Argument $_ }) + Write-Output ("& " + "'" + $python.Replace("'", "''") + "' " + ($formatted -join " ")) + } else { + & $python @install + } +} + +Invoke-EwoksInstall $args diff --git a/src/ewoks/_bootstrap/ewoks-install.sh b/src/ewoks/_bootstrap/ewoks-install.sh new file mode 100644 index 0000000..565ed78 --- /dev/null +++ b/src/ewoks/_bootstrap/ewoks-install.sh @@ -0,0 +1,441 @@ +#!/bin/sh +# Install the python environment of an ewoks workflow with nothing but a package +# manager. For example +# +# curl -LsSf https://ewoks.readthedocs.io/en/stable/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv +# +set -eu + +usage() { + cat <<'EOF' +Usage: ewoks-install.sh [OPTIONS] WORKFLOW [ARGS ...] + +Install the python environment of an ewoks workflow with nothing but a package +manager. The ewoks version that generated the workflow requirements, and the +engine that saved the workflow, are installed in a bootstrap environment, which +then runs + + ewoks install WORKFLOW ARGS + +See 'ewoks install --help' for ARGS. The package manager that creates the +bootstrap environment is selected with '--package-manager-name' and +'--package-manager-command', which are passed to 'ewoks install' as well. +Without them it is the first available of uv, pixi, conda, poetry and pip-venv. +A package manager that is not installed can be installed in the bootstrap +directory with '--install-package-manager'. + +Options: + --print-command Print the 'ewoks install' command instead of + running it. + --ewoks-requirement REQ Install REQ instead of the ewoks version and engine + of the workflow, for example 'ewoks>=7' or a wheel + file. Repeat it to install more, for example + '--ewoks-requirement ewoks --ewoks-requirement + ewoksmyengine'. + --bootstrap-dir DIR Directory of the bootstrap environments. + Default: ~/.ewoks/bootstrap + --install-package-manager + Install the package manager of + '--package-manager-name' when it is not installed. + -h, --help Show this help. +EOF +} + +say() { + printf 'ewoks-install: %s\n' "$*" >&2 +} + +die() { + say "error: $*" + exit 1 +} + +# Value in the requirements of a workflow file, for example `ewoks.version`. The +# requirements are JSON, which can be embedded in another format, or YAML. +requirement_value() { + value=$(json_requirement_value "$1" "$2") + [ -n "$value" ] || value=$(yaml_requirement_value "$1" "$2") + printf '%s' "$value" +} + +json_requirement_value() { + q="[\"']" + s='[[:space:]]*' + value="${q}$s:$s${q}\([^\"']*\)${q}" + # The members of an object, which can contain objects themselves + members='[^{}]*\({[^{}]*}[^{}]*\)*' + case $2 in + ewoks.version) + pattern="${q}ewoks${q}$s:$s{$members${q}version$value" group=2 + ;; + ewoks.engine.name) + pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}name$value" group=1 + ;; + ewoks.engine.version) + pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}version$value" group=1 + ;; + python.version) + pattern="${q}python${q}$s:$s{[^{}]*${q}version$value" group=1 + ;; + esac + tr -d '\n\r' <"$1" | sed 's/"/"/g' | sed -n "s/.*$pattern.*/\\$group/p" +} + +yaml_requirement_value() { + awk -v wanted="requirements.$2" -v sq="'" ' + { + sub(/\r$/, "") + if ($0 ~ /^[ \t]*(#|$)/) next + # The dash of a list item belongs to the indentation of its first key + match($0, /^[ -]*/) + indent = RLENGTH + rest = substr($0, indent + 1) + if (rest !~ /^[A-Za-z0-9_]+:/) next + key = rest + sub(/:.*/, "", key) + value = rest + sub(/^[^:]*:[ \t]*/, "", value) + while (depth > 0 && indents[depth] >= indent) depth-- + depth++ + indents[depth] = indent + keys[depth] = key + path = keys[1] + for (i = 2; i <= depth; i++) path = path "." keys[i] + n = length(path) - length(wanted) + if (value == "" || n < 0 || substr(path, n + 1) != wanted) next + if (n > 0 && substr(path, n, 1) != ".") next + gsub("^[\"" sq "]|[\"" sq "]$", "", value) + found = value + } + END { printf "%s", found } + ' "$1" +} + +# Version of an installed distribution, or nothing when it is not installed +distribution_version() { + "$1" -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' "$2" 2>/dev/null || true +} + +# Argument quoted for the shell when needed +quote() { + case $1 in + '' | *[!A-Za-z0-9_./=:@%+,-]*) + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" + ;; + *) + printf '%s' "$1" + ;; + esac +} + +# Python interpreter with this major.minor version, or the default one +find_python() { + if [ -n "$1" ] && command -v "python$1" >/dev/null 2>&1; then + printf 'python%s' "$1" + elif command -v python3 >/dev/null 2>&1; then + printf 'python3' + else + die "no python interpreter found" + fi +} + +# Download a URL to a file +download() { + if command -v curl >/dev/null 2>&1; then + curl -LsSf -o "$2" "$1" + elif command -v wget >/dev/null 2>&1; then + wget -q -O "$2" "$1" + else + die "curl or wget is needed to download $1" + fi +} + +# Executable of a package manager installed in a directory by this script +manager_executable() { + case $1 in + uv) printf '%s/bin/uv' "$2" ;; + pixi) printf '%s/bin/pixi' "$2" ;; + poetry) printf '%s/bin/poetry' "$2" ;; + conda) printf '%s/bin/conda' "$2" ;; + esac +} + +# Install a package manager in a directory with its official installer +install_manager() { + say "install $1 in $2" + rm -rf "$2" + mkdir -p "$(dirname "$2")" + # The Miniforge installer requires the '.sh' suffix + installer_dir=$(mktemp -d) + installer=$installer_dir/installer.sh + case $1 in + uv) + download https://astral.sh/uv/install.sh "$installer" + UV_INSTALL_DIR=$2/bin UV_NO_MODIFY_PATH=1 sh "$installer" >&2 + ;; + pixi) + download https://pixi.sh/install.sh "$installer" + PIXI_HOME=$2 PIXI_NO_PATH_UPDATE=1 sh "$installer" >&2 + ;; + poetry) + download https://install.python-poetry.org "$installer" + POETRY_HOME=$2 "$(find_python "")" "$installer" >&2 + ;; + conda) + system=$(uname -s) + [ "$system" != Darwin ] || system=MacOSX + download "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$system-$(uname -m).sh" "$installer" + bash "$installer" -b -p "$2" >&2 + ;; + esac + rm -rf "$installer_dir" +} + +# The whole script is parsed before it runs, which matters with `curl | sh` +main() { + # --- Arguments: the bootstrap options are removed, the others are kept --- + + print_command=0 + install_package_manager=0 + # Quoted for the shell + requirements="" + root=$HOME/.ewoks/bootstrap + manager="" + manager_command="" + workflow="" + remaining=$# + while [ "$remaining" -gt 0 ]; do + arg=$1 + shift + remaining=$((remaining - 1)) + case $arg in + -h | --help) + usage + exit 0 + ;; + --print-command) + print_command=1 + continue + ;; + --install-package-manager) + install_package_manager=1 + continue + ;; + --ewoks-requirement) + [ "$remaining" -gt 0 ] || die "$arg requires a value" + requirements="$requirements $(quote "$1")" + shift + remaining=$((remaining - 1)) + continue + ;; + --ewoks-requirement=*) + requirements="$requirements $(quote "${arg#*=}")" + continue + ;; + --bootstrap-dir) + [ "$remaining" -gt 0 ] || die "$arg requires a value" + root=$1 + shift + remaining=$((remaining - 1)) + continue + ;; + --bootstrap-dir=*) + root=${arg#*=} + continue + ;; + esac + set -- "$@" "$arg" + case $arg in + --package-manager-name | --package-manager-command) + [ "$remaining" -gt 0 ] || die "$arg requires a value" + value=$1 + shift + remaining=$((remaining - 1)) + set -- "$@" "$value" + if [ "$arg" = --package-manager-name ]; then + manager=$value + else + manager_command=$value + fi + ;; + --package-manager-name=*) + manager=${arg#*=} + ;; + --package-manager-command=*) + manager_command=${arg#*=} + ;; + -*) ;; + *) + if [ -z "$workflow" ] && [ -f "$arg" ]; then + workflow=$arg + fi + ;; + esac + done + + if [ -z "$workflow" ]; then + usage >&2 + die "provide the file of the workflow" + fi + + # --- Package manager --- + + manager=$(printf '%s' "$manager" | tr '[:upper:]' '[:lower:]') + if [ "$install_package_manager" = 1 ] && [ -z "$manager" ]; then + die "--install-package-manager requires --package-manager-name" + fi + if [ -z "$manager" ]; then + [ -z "$manager_command" ] || die "--package-manager-command requires --package-manager-name" + for name in uv pixi conda poetry; do + if command -v "$name" >/dev/null 2>&1; then + manager=$name + break + fi + done + manager=${manager:-pip-venv} + fi + + case $manager in + uv | pixi | conda | poetry) cmd=${manager_command:-$manager} ;; + pip-venv) cmd=${manager_command:-} ;; + *) die "package manager '$manager' is not supported" ;; + esac + + # `ewoks install` needs the package manager this script installed + if [ -n "$cmd" ] && [ -z "$manager_command" ] && ! command -v "$cmd" >/dev/null 2>&1; then + manager_dir=$root/package-managers/$manager + cmd=$(manager_executable "$manager" "$manager_dir") + if [ ! -x "$cmd" ]; then + [ "$install_package_manager" = 1 ] || + die "$manager is not installed (see --install-package-manager)" + install_manager "$manager" "$manager_dir" + [ -x "$cmd" ] || die "cannot install $manager in $manager_dir" + fi + set -- "$@" --package-manager-command "$cmd" + fi + + # --- Ewoks, engine and python versions of the workflow requirements --- + + version=$(requirement_value "$workflow" ewoks.version) + engine=$(requirement_value "$workflow" ewoks.engine.name) + engine_version=$(requirement_value "$workflow" ewoks.engine.version) + python_version=$(requirement_value "$workflow" python.version) + python_major_minor=$(printf '%s' "$python_version" | sed -n 's/^\([0-9]*\.[0-9]*\).*/\1/p') + [ -n "$engine_version" ] || engine="" + + # The bootstrap environment of pinned requirements is up to date when it has their versions + pinned=0 + if [ -n "$requirements" ]; then + tag=custom-$(printf '%s' "$requirements" | cksum | cut -d ' ' -f 1) + elif [ -n "$version" ]; then + pinned=1 + tag=$version + requirements=$(quote "ewoks==$version") + if [ -n "$engine" ]; then + tag=$tag-$engine-$engine_version + requirements="$requirements $(quote "$engine==$engine_version")" + fi + else + say "the workflow does not provide the ewoks version: using the latest one" + tag=latest + requirements=$(quote "ewoks>=7") + fi + requirements=${requirements# } + + # pip-venv and poetry cannot provide a python version: the interpreter decides + case $manager in + pip-venv) + if [ -z "$cmd" ]; then + cmd=$(find_python "$python_major_minor") + fi + # shellcheck disable=SC2086 + python_major_minor=$($cmd -c 'import sys; print("%d.%d" % sys.version_info[:2])') + ;; + poetry) + base_python=$(find_python "$python_major_minor") + python_major_minor=$("$base_python" -c 'import sys; print("%d.%d" % sys.version_info[:2])') + ;; + esac + + location=$root/$manager/ewoks-$tag${python_major_minor:+-python$python_major_minor} + + # --- Bootstrap environment --- + + case $manager in + pixi) python=$location/.pixi/envs/default/bin/python ;; + poetry) python=$location/.venv/bin/python ;; + *) python=$location/bin/python ;; + esac + + # shellcheck disable=SC2086 + if [ ! -x "$python" ]; then + say "create the bootstrap environment $location" + # A previous attempt failed + rm -rf "$location" + mkdir -p "$location" + case $manager in + uv) + $cmd venv --quiet ${python_major_minor:+--python "$python_major_minor"} "$location" >&2 + ;; + pixi) + $cmd init "$location" >&2 + $cmd add --manifest-path "$location/pixi.toml" "python${python_major_minor:+=$python_major_minor}" pip >&2 + ;; + conda) + $cmd create --yes --quiet --prefix "$location" "python${python_major_minor:+=$python_major_minor}" pip >&2 + ;; + poetry) + printf '[tool.poetry]\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = "*"\n' >"$location/pyproject.toml" + interpreter=$("$base_python" -c 'import sys; print(sys.executable)') + # Poetry uses an active virtual or conda environment instead of the one of the project. + # Without its own python it runs the `python` command, which can be python 2. + ( + unset VIRTUAL_ENV CONDA_PREFIX + POETRY_VIRTUALENVS_IN_PROJECT=true POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true \ + $cmd -C "$location" env use "$interpreter" >&2 + ) + ;; + pip-venv) + $cmd -m venv "$location" >&2 + ;; + esac + fi + + up_to_date=0 + if [ "$pinned" = 1 ] && [ "$(distribution_version "$python" ewoks)" = "$version" ]; then + if [ -z "$engine" ] || [ "$(distribution_version "$python" "$engine")" = "$engine_version" ]; then + up_to_date=1 + fi + fi + + if [ "$up_to_date" = 0 ]; then + say "install $requirements in $location" + # The requirements are quoted and `cmd` can have arguments + if [ "$manager" = uv ]; then + eval "\$cmd pip install --quiet --upgrade --python \"\$python\" $requirements" >&2 || + die "cannot install $requirements (see --ewoks-requirement)" + else + eval "\"\$python\" -m pip install --quiet --disable-pip-version-check --upgrade $requirements" >&2 || + die "cannot install $requirements (see --ewoks-requirement)" + fi + fi + + # --- ewoks install --- + + set -- "$python" -m ewoks install "$@" + + if [ "$print_command" = 1 ]; then + command="" + for arg in "$@"; do + command="$command $(quote "$arg")" + done + printf '%s\n' "${command# }" + elif [ ! -t 0 ] && (exec /dev/null; then + # Confirmation prompts do not read stdin, which is this script with `curl | sh` + exec "$@" Optional[Tuple[str, str]]: + """Name and version of the distribution that provides the engine.""" + module = type(engine).__module__ + for distribution in importlib.metadata.distributions(): + for entry_point in distribution.entry_points: + if ( + entry_point.group == "ewoks.engines" + and entry_point.value.split(":")[0] == module + ): + return distribution.metadata["Name"], distribution.version + return None + + _EngineClassLoaderGenerator = Generator[ Tuple[str, Callable[[], Type[WorkflowEngine]]], None, None ] diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index b58ff9c..475e288 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -21,6 +21,7 @@ from .utils.detect import get_manager from .utils.environment import Environment from .utils.metadata import last_resort +from .utils.metadata.models import EngineInfo logger = logging.getLogger(__file__) @@ -49,12 +50,16 @@ def add_requirements( graph: TaskGraph, manager_name: Optional[str] = None, manager_command: Tuple[str, ...] = tuple(), + engine: Optional[Tuple[str, str]] = None, ) -> str: """Add requirements to a workflow definition in-place. Returns the name of the - package manager that generated them. + package manager that generated them. `engine` is the name and version of the + distribution of the engine that saves the workflow. """ manager = get_manager(manager_name=manager_name, manager_command=manager_command) requirements = manager.gather_requirements() + if requirements.ewoks and engine: + requirements.ewoks.engine = EngineInfo(name=engine[0], version=engine[1]) graph.graph.graph["requirements"] = requirements.model_dump() return manager.NAME diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 2ae0ef6..b87575f 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -1,3 +1,4 @@ +import importlib.metadata import logging from abc import abstractmethod from pathlib import Path @@ -45,6 +46,13 @@ class BaseManagerInfo(models.BaseModel): class BaseRequirements(models.BaseModel): + ewoks: Optional[models.EwoksInfo] = Field( + default=None, + description=( + "Ewoks that generated the requirements. Not set for requirements that " + "ewoks did not generate." + ), + ) system: models.SystemInfo = Field( description="Operating system on which the requirements were generated." ) @@ -170,7 +178,8 @@ def gather_requirements(self) -> BaseRequirements: ) manager = dict(name=self.NAME, version=manager_version, files=files) - return self.REQUIREMENTS_MODEL(manager=manager, **metadata) + ewoks = dict(version=importlib.metadata.version("ewoks")) + return self.REQUIREMENTS_MODEL(ewoks=ewoks, manager=manager, **metadata) def environment(self, location: Union[str, Path]) -> Environment: """Environment the manager creates at this location.""" diff --git a/src/ewoks/_requirements/utils/metadata/models.py b/src/ewoks/_requirements/utils/metadata/models.py index 72bfd66..8c333fe 100644 --- a/src/ewoks/_requirements/utils/metadata/models.py +++ b/src/ewoks/_requirements/utils/metadata/models.py @@ -5,6 +5,22 @@ from pydantic import Field +class EngineInfo(BaseModel): + name: str = Field(description="Distribution name.", examples=["ewoksorange"]) + version: str = Field(description="Distribution version.", examples=["2.2.0"]) + + +class EwoksInfo(BaseModel): + version: str = Field(description="Ewoks version.", examples=["7.0.0"]) + engine: Optional[EngineInfo] = Field( + default=None, + description=( + "Distribution of the engine that saved the workflow, which is needed " + "to load it." + ), + ) + + class SystemInfo(BaseModel): system: str = Field(description="Operating system name.", examples=["Linux"]) release: str = Field( diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index 91d9f2f..ecfa770 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -222,10 +222,14 @@ def convert_graph( elif isinstance(package_manager_command, str): package_manager_command = _split_command(package_manager_command) + engine, _ = _engines.get_serialization_engine( + destination, representation=save_options.get("representation") + ) manager_name = _requirements.add_requirements( graph, manager_name=package_manager_name, manager_command=package_manager_command, + engine=_engines.get_engine_distribution(engine), ) logger.info("Requirements generated with the %r package manager", manager_name) else: diff --git a/src/ewoks/tests/conftest.py b/src/ewoks/tests/conftest.py index da09d6c..587bdbb 100644 --- a/src/ewoks/tests/conftest.py +++ b/src/ewoks/tests/conftest.py @@ -3,6 +3,13 @@ import pytest +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "package_manager_install: installs package managers from the internet", + ) + + @pytest.fixture def mock_icat_client(monkeypatch): mock_defaults = MagicMock() diff --git a/src/ewoks/tests/requirements/test_bootstrap.py b/src/ewoks/tests/requirements/test_bootstrap.py new file mode 100644 index 0000000..9af91f2 --- /dev/null +++ b/src/ewoks/tests/requirements/test_bootstrap.py @@ -0,0 +1,313 @@ +"""Tests of the scripts that run `ewoks install` with nothing but a package manager. + +The bootstrap environment is prepared in advance so the scripts do not install +anything: its python interpreter runs the ewoks under test. Except for the tests +marked `package_manager_install`, which install package managers and ewoks from +the internet. +""" + +import html +import importlib.metadata +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Dict +from typing import List +from typing import Optional + +import pytest +import yaml + +from .utils import PYTHON_MINOR_VERSION +from .utils import PYTHON_VERSION +from .utils import manager_requirements + +_SCRIPTS = Path(__file__).parents[2] / "_bootstrap" +_EWOKS_VERSION = importlib.metadata.version("ewoks") +_INSTALLABLE_MANAGERS = ["uv", "pixi", "poetry", "conda"] + + +@pytest.fixture(params=["sh", "powershell"]) +def bootstrap_script(request) -> List[str]: + """Command that runs a bootstrap script.""" + if sys.platform == "win32": + pytest.skip("the bootstrap environment of the tests needs a POSIX shell") + if request.param == "sh": + return ["sh", str(_SCRIPTS / "ewoks-install.sh")] + pwsh = shutil.which("pwsh") + if pwsh is None: + pytest.skip("PowerShell is not installed") + return [pwsh, "-NoProfile", "-File", str(_SCRIPTS / "ewoks-install.ps1")] + + +@pytest.fixture(params=["sh", "powershell"]) +def any_platform_script(request) -> List[str]: + """Command that runs a bootstrap script on any platform.""" + if request.param == "sh": + if sys.platform == "win32": + pytest.skip("sh is not available on Windows") + return ["sh", str(_SCRIPTS / "ewoks-install.sh")] + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell is not installed") + return [ + powershell, + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(_SCRIPTS / "ewoks-install.ps1"), + ] + + +@pytest.fixture +def no_managers_env(tmp_path) -> Dict[str, str]: + """Environment variables with a PATH that only provides python and the + system tools.""" + if sys.platform == "win32": + system_root = os.environ.get("SystemRoot", r"C:\Windows") + directories = [ + os.path.dirname(sys.executable), + os.path.join(system_root, "System32"), + system_root, + os.path.join(system_root, "System32", "WindowsPowerShell", "v1.0"), + ] + else: + bin_dir = tmp_path / "python-bin" + bin_dir.mkdir() + for name in ("python3", f"python{PYTHON_MINOR_VERSION}"): + python = bin_dir / name + with open(python, "w", encoding="utf-8") as fh: + fh.write(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + python.chmod(0o755) + directories = [str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"] + path = os.pathsep.join(directories) + for manager in _INSTALLABLE_MANAGERS: + if shutil.which(manager, path=path): + pytest.skip(f"{manager} is installed in a system directory") + env = { + name: value + for name, value in os.environ.items() + if name not in ("VIRTUAL_ENV", "CONDA_PREFIX") + } + env["PATH"] = path + return env + + +@pytest.fixture +def bootstrap_dir(tmp_path) -> Path: + """Directory of the bootstrap environments with the environment of the + ewoks under test for pip-venv.""" + root = tmp_path / "bootstrap" + _bootstrap_python(root, _EWOKS_VERSION) + return root + + +def test_print_command(bootstrap_script, bootstrap_dir, tmp_path): + """The bootstrap environment of the ewoks version of the workflow runs + `ewoks install` with the arguments that are not for the script.""" + workflow = _workflow(tmp_path) + arguments = [str(workflow), *_pip_venv_arguments(), "--yes"] + + result = _run(bootstrap_script, ["--print-command", *arguments], bootstrap_dir) + + assert result.returncode == 0, result.stderr + command = result.stdout.strip() + python = ( + bootstrap_dir + / "pip-venv" + / f"ewoks-{_EWOKS_VERSION}-python{PYTHON_MINOR_VERSION}" + / "bin" + / "python" + ) + assert str(python) in command + assert command.endswith(f"-m ewoks install {' '.join(arguments)}") + + +def test_run_ewoks_install(bootstrap_script, bootstrap_dir, tmp_path): + """`ewoks install` runs in the bootstrap environment.""" + workflow = _workflow(tmp_path) + arguments = [str(workflow), *_pip_venv_arguments(), "--yes", "--in-place"] + + result = _run(bootstrap_script, arguments, bootstrap_dir) + + # The requirements of the workflow have nothing to install + assert "No distributions provided to install" in result.stderr + assert f"Install failed for {workflow}" in result.stdout + + +def test_no_workflow(bootstrap_script, bootstrap_dir): + result = _run(bootstrap_script, ["--test", "demo"], bootstrap_dir) + + assert result.returncode != 0 + assert "provide the file of the workflow" in result.stderr + + +def test_unsupported_manager(bootstrap_script, bootstrap_dir, tmp_path): + workflow = _workflow(tmp_path) + + result = _run( + bootstrap_script, + [str(workflow), "--package-manager-name", "unknown"], + bootstrap_dir, + ) + + assert result.returncode != 0 + assert "package manager 'unknown' is not supported" in result.stderr + + +def test_install_manager_requires_name(any_platform_script, tmp_path): + workflow = _workflow(tmp_path) + + result = _run( + any_platform_script, + [str(workflow), "--install-package-manager"], + tmp_path / "bootstrap", + ) + + assert result.returncode != 0 + assert "--install-package-manager requires --package-manager-name" in ( + result.stderr + ) + + +def test_manager_not_installed(any_platform_script, no_managers_env, tmp_path): + workflow = _workflow(tmp_path) + + result = _run( + any_platform_script, + [str(workflow), "--package-manager-name", "pixi"], + tmp_path / "bootstrap", + env=no_managers_env, + ) + + assert result.returncode != 0 + assert "pixi is not installed (see --install-package-manager)" in result.stderr + + +@pytest.mark.package_manager_install +@pytest.mark.parametrize("manager", _INSTALLABLE_MANAGERS) +def test_install_manager(any_platform_script, no_managers_env, tmp_path, manager): + """The package manager is installed in the bootstrap directory, used again + by the next run and passed to `ewoks install`.""" + workflow = _workflow(tmp_path) + bootstrap_dir = tmp_path / "bootstrap" + arguments = [ + "--print-command", + "--ewoks-requirement", + "ewoks", + str(workflow), + "--package-manager-name", + manager, + "--yes", + ] + + result = _run( + any_platform_script, + ["--install-package-manager", *arguments], + bootstrap_dir, + env=no_managers_env, + timeout=1800, + ) + + assert result.returncode == 0, result.stderr + assert f"install {manager} in" in result.stderr + manager_dir = bootstrap_dir / "package-managers" / manager + command = result.stdout.strip() + assert f"--package-manager-command {manager_dir}" in command.replace("'", "") + + result = _run(any_platform_script, arguments, bootstrap_dir, env=no_managers_env) + + assert result.returncode == 0, result.stderr + assert f"install {manager} in" not in result.stderr + assert result.stdout.strip() == command + + +@pytest.mark.parametrize("workflow_format", ["json", "yaml", "ows"]) +def test_requirements_of_any_format(bootstrap_script, tmp_path, workflow_format): + """The bootstrap environment of the ewoks version and the engine of the + workflow is used, whatever the format of the workflow.""" + engine_version = importlib.metadata.version("ewokscore") + engine = {"name": "ewokscore", "version": engine_version} + workflow = _workflow(tmp_path, workflow_format, engine=engine) + bootstrap_dir = tmp_path / "bootstrap" + python = _bootstrap_python( + bootstrap_dir, f"{_EWOKS_VERSION}-ewokscore-{engine_version}" + ) + + result = _run( + bootstrap_script, + ["--print-command", str(workflow), *_pip_venv_arguments()], + bootstrap_dir, + ) + + assert result.returncode == 0, result.stderr + assert "install ewoks" not in result.stderr + assert str(python) in result.stdout + + +def _bootstrap_python(root: Path, tag: str) -> Path: + """Python interpreter of a pip-venv bootstrap environment that runs the ewoks + under test.""" + location = root / "pip-venv" / f"ewoks-{tag}-python{PYTHON_MINOR_VERSION}" + python = location / "bin" / "python" + python.parent.mkdir(parents=True) + with open(python, "w", encoding="utf-8") as fh: + fh.write(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + python.chmod(0o755) + return python + + +def _workflow( + tmp_path: Path, workflow_format: str = "json", engine: Optional[dict] = None +) -> Path: + requirements = manager_requirements("pip-venv", python_version=PYTHON_VERSION) + requirements["ewoks"] = {"version": _EWOKS_VERSION, "engine": engine} + graph = { + "graph": {"id": "bootstrap", "requirements": requirements}, + "nodes": [], + "links": [], + } + filename = tmp_path / f"workflow.{workflow_format}" + with open(filename, "w", encoding="utf-8") as fh: + if workflow_format == "json": + # The engine comes before the ewoks version + json.dump(graph, fh, indent=2, sort_keys=True) + elif workflow_format == "yaml": + yaml.safe_dump(graph, fh) + else: + # Orange workflows embed the graph attributes as JSON + graph_attrs = html.escape(json.dumps(graph["graph"]), quote=False) + fh.write( + f"\n{graph_attrs}\n\n" + ) + return filename + + +def _pip_venv_arguments() -> List[str]: + return [ + "--package-manager-name", + "pip-venv", + "--package-manager-command", + sys.executable, + ] + + +def _run( + script: List[str], + arguments: List[str], + bootstrap_dir: Path, + env: Optional[Dict[str, str]] = None, + timeout: float = 300, +) -> "subprocess.CompletedProcess[str]": + return subprocess.run( # noqa: S603 - Script under test + [*script, "--bootstrap-dir", str(bootstrap_dir), *arguments], + capture_output=True, + text=True, + env=env, + stdin=subprocess.DEVNULL, + timeout=timeout, + ) diff --git a/src/ewoks/tests/requirements/test_managers.py b/src/ewoks/tests/requirements/test_managers.py index 402935c..07d55b5 100644 --- a/src/ewoks/tests/requirements/test_managers.py +++ b/src/ewoks/tests/requirements/test_managers.py @@ -1,5 +1,6 @@ """Tests that apply to all package managers.""" +import importlib.metadata from typing import Iterator import pytest @@ -83,6 +84,7 @@ def test_gather_requirements(manager, manager_case, monkeypatch): requirements = manager.gather_requirements() assert isinstance(requirements, manager_case.MANAGER_CLS.REQUIREMENTS_MODEL) + assert requirements.ewoks.version == importlib.metadata.version("ewoks") assert requirements.manager.name == manager_case.NAME assert requirements.manager.version assert requirements.python.version == PYTHON_VERSION diff --git a/src/ewoks/tests/test_convert_cli.py b/src/ewoks/tests/test_convert_cli.py index dc814d1..911e331 100644 --- a/src/ewoks/tests/test_convert_cli.py +++ b/src/ewoks/tests/test_convert_cli.py @@ -1,3 +1,5 @@ +import importlib.metadata +import json import os import sys @@ -34,6 +36,10 @@ def test_convert_to_json(graph_name, tmpdir): graph = load_graph(destination) assert_in_graph_requirements(graph, "ewokscore") + assert graph.graph.graph["requirements"]["ewoks"]["engine"] == { + "name": "ewokscore", + "version": importlib.metadata.version("ewokscore"), + } def test_convert_with_all_inputs(tmpdir): @@ -133,6 +139,12 @@ def test_convert_to_ows(graph_name, tmpdir): tree = ElementTree.parse(destination) root = tree.getroot() + graph_attrs = json.loads(root.find("./ewoks_graph_attrs").text) + assert graph_attrs["requirements"]["ewoks"]["engine"] == { + "name": "ewoksorange", + "version": importlib.metadata.version("ewoksorange"), + } + for node in root.findall("./nodes/node"): try: assert issubclass(import_qualname(node.get("qualified_name")), OWBaseWidget)