Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ dependencies = []
datasource-kit = "datasource_kit.cli:main"

[project.optional-dependencies]
# Blob/artifact resolution and in-process journal recording only.
# Fala is GitHub-only (name = "fala"); the PEP 508 URL keeps pip off the
# unrelated PyPI project of the same name. Adapter needs record_in_process
# (Fala 0.7.28+).
Expand Down
61 changes: 2 additions & 59 deletions src/datasource_kit/adapters/fala.py
Original file line number Diff line number Diff line change
@@ -1,52 +1,22 @@
"""Optional Fala execution and artifact adapters."""

from __future__ import annotations

from collections.abc import Callable, Mapping
from io import BytesIO
from os import PathLike
from pathlib import Path
from typing import Any, BinaryIO, Protocol, TypeVar
from typing import Any, Protocol, TypeVar

from datasource_kit.execution import ExecutionRequest
from datasource_kit.protocols import ArtifactStore

__all__ = ["FalaArtifactStore", "FalaExecutionBackend"]
__all__ = ["FalaExecutionBackend"]

_T = TypeVar("_T")

_INSTALL_HINT = (
"FalaArtifactStore requires the 'fala' extra: pip install datasource-kit[fala]"
)
_EXECUTION_INSTALL_HINT = (
"FalaExecutionBackend requires the 'fala' extra: pip install datasource-kit[fala]"
)
_EXECUTION_CAPABILITY_ERROR = (
"FalaExecutionBackend requires Fala 0.7.28 or newer with the public "
"record_in_process API"
)
_BLOB_KIND = "blob"
_PAYLOAD_FILENAME = "payload.bin"


class _FalaArtifactRef(Protocol):
uri: str


class _FalaArtifactRefFactory(Protocol):
def __call__(self, *, kind: str, uri: str) -> object: ...


class _FalaFileArtifactStore(Protocol):
def put_fileobj(
self,
*,
kind: str,
fileobj: BinaryIO,
filename: str,
) -> _FalaArtifactRef: ...

def resolve(self, artifact: object) -> Path: ...


class _RecordInProcess(Protocol):
Expand Down Expand Up @@ -99,30 +69,3 @@ def execute(
inputs=request.inputs,
metadata=request.metadata,
)


class FalaArtifactStore(ArtifactStore):
"""Thin ``ArtifactStore`` adapter over Fala's file artifact store."""

def __init__(self, root: str | PathLike[str]) -> None:
try:
from fala.artifacts import ArtifactRef, FileArtifactStore
except ImportError as exc:
raise ImportError(_INSTALL_HINT) from exc

self._artifact_ref: _FalaArtifactRefFactory = ArtifactRef
self._store: _FalaFileArtifactStore = FileArtifactStore(Path(root))

def store(self, payload: bytes) -> str:
"""Store bytes in Fala and return the content-addressed artifact URI."""
artifact = self._store.put_fileobj(
kind=_BLOB_KIND,
fileobj=BytesIO(payload),
filename=_PAYLOAD_FILENAME,
)
return artifact.uri

def resolve(self, ref: str) -> bytes:
"""Resolve a Fala artifact URI and return the payload bytes."""
artifact = self._artifact_ref(kind=_BLOB_KIND, uri=ref)
return self._store.resolve(artifact).read_bytes()
83 changes: 10 additions & 73 deletions tests/adapters/test_fala_artifact_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,12 @@
import builtins
import hashlib
import sys
import types
from pathlib import Path
from typing import BinaryIO, Final

import pytest


_INSTALL_HINT: Final = (
"FalaArtifactStore requires the 'fala' extra: "
"pip install datasource-kit[fala]"
)


def _block_fala_imports(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delitem(sys.modules, "fala", raising=False)
monkeypatch.delitem(sys.modules, "fala.artifacts", raising=False)
real_import = builtins.__import__

def guarded_import(
Expand All @@ -35,50 +25,6 @@ def guarded_import(
monkeypatch.setattr(builtins, "__import__", guarded_import)


class _FakeArtifactRef:
def __init__(self, *, kind: str, uri: str) -> None:
self.kind = kind
self.uri = uri


class _FakeFileArtifactStore:
def __init__(self, root: str | Path) -> None:
self.root = Path(root)
self._payloads: dict[str, bytes] = {}

def put_fileobj(
self,
*,
kind: str,
fileobj: BinaryIO,
filename: str,
artifact_id: str | None = None,
metadata: dict[str, object] | None = None,
) -> _FakeArtifactRef:
payload = fileobj.read()
digest = hashlib.sha256(payload).hexdigest()
uri = f"fala-artifact://sha256/{digest}"
self._payloads[uri] = payload
return _FakeArtifactRef(kind=kind, uri=uri)

def resolve(self, artifact: _FakeArtifactRef) -> Path:
digest = artifact.uri.removeprefix("fala-artifact://sha256/")
target = self.root / "blobs" / "sha256" / digest[:2] / digest
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(self._payloads[artifact.uri])
return target


def _install_fake_fala(monkeypatch: pytest.MonkeyPatch) -> None:
package = types.ModuleType("fala")
artifacts = types.ModuleType("fala.artifacts")
artifacts.ArtifactRef = _FakeArtifactRef
artifacts.FileArtifactStore = _FakeFileArtifactStore
package.artifacts = artifacts
monkeypatch.setitem(sys.modules, "fala", package)
monkeypatch.setitem(sys.modules, "fala.artifacts", artifacts)


def test_importing_core_and_adapters_when_fala_missing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
Expand All @@ -94,36 +40,27 @@ def test_importing_core_and_adapters_when_fala_missing(
assert adapters.__name__ == "datasource_kit.adapters"


def test_fala_artifact_store_requires_fala_extra_when_missing(
def test_fala_adapter_does_not_export_artifact_store(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Given: the adapter module is importable, but fala itself is not.
from datasource_kit.adapters.fala import FalaArtifactStore

_block_fala_imports(monkeypatch)
from datasource_kit.adapters import fala

# When / Then: constructing the adapter raises the install hint.
with pytest.raises(ImportError) as exc_info:
FalaArtifactStore("unused")
assert str(exc_info.value) == _INSTALL_HINT
with pytest.raises(ImportError):
from datasource_kit.adapters.fala import FalaArtifactStore
assert "FalaArtifactStore" not in fala.__all__


def test_fala_artifact_store_round_trips_bytes_through_fala(
def test_artifact_store_round_trips_bytes_without_fala(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
# Given: a fala-compatible FileArtifactStore implementation is importable.
_install_fake_fala(monkeypatch)
from datasource_kit import ArtifactStore
from datasource_kit.adapters.fala import FalaArtifactStore

store = FalaArtifactStore(tmp_path)
_block_fala_imports(monkeypatch)
from datasource_kit import ArtifactStore, InMemoryArtifactStore

# When: payload bytes are stored and resolved through the adapter.
store = InMemoryArtifactStore()
ref = store.store(b"payload")
resolved = store.resolve(ref)

# Then: the adapter satisfies the kit protocol and returns the original bytes.
assert isinstance(store, ArtifactStore)
assert ref.startswith("fala-artifact://sha256/")
assert ref == hashlib.sha256(b"payload").hexdigest()
assert resolved == b"payload"
Loading