content-addressed blob storage: blob_storage.py, memory, local, and S3 adapters

src/kontextual_engine/adapters/local_files/__init__.py

@@ -1,5 +1,6 @@
 """Local filesystem ingestion connector."""
 
+from .blob_storage import LocalBlobStorage
 from .connector import LocalFileConnector
 
-__all__ = ["LocalFileConnector"]
+__all__ = ["LocalBlobStorage", "LocalFileConnector"]

src/kontextual_engine/adapters/local_files/blob_storage.py

@@ -0,0 +1,141 @@
+"""Local filesystem content-addressed blob storage."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+
+from kontextual_engine.core import new_id
+from kontextual_engine.errors import NotFoundError, ValidationError
+from kontextual_engine.ports import BlobCleanupResult, BlobRef, BlobWriteResult, blob_digest, digest_storage_key
+
+
+class LocalBlobStorage:
+    adapter_name = "local"
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root).expanduser().resolve()
+
+    def put_bytes(self, content: bytes, *, media_type: str | None = None) -> BlobWriteResult:
+        digest = blob_digest(content)
+        storage_key = digest_storage_key(digest)
+        path = self._path(storage_key)
+        storage_ref = self._storage_ref(storage_key)
+        created = not path.exists()
+        if created:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            tmp = path.parent / f".{path.name}.{new_id('tmp')}"
+            tmp.write_bytes(content)
+            actual = blob_digest(tmp.read_bytes())
+            if actual != digest:
+                tmp.unlink(missing_ok=True)
+                raise ValidationError("Blob digest verification failed", details={"expected": digest, "actual": actual})
+            tmp.replace(path)
+        else:
+            existing = path.read_bytes()
+            actual = blob_digest(existing)
+            if actual != digest or len(existing) != len(content):
+                raise ValidationError(
+                    "Existing blob digest mismatch",
+                    details={"storage_ref": storage_ref, "expected": digest, "actual": actual},
+                )
+        return BlobWriteResult(
+            BlobRef(
+                digest=digest,
+                size_bytes=len(content),
+                storage_key=storage_key,
+                storage_ref=storage_ref,
+                adapter=self.adapter_name,
+                media_type=media_type,
+            ),
+            created=created,
+        )
+
+    def read_bytes(self, storage_ref: str) -> bytes:
+        path = self._path(self._storage_key(storage_ref))
+        if not path.exists():
+            raise NotFoundError("Blob not found", details={"storage_ref": storage_ref})
+        return path.read_bytes()
+
+    def iter_bytes(self, storage_ref: str, *, chunk_size: int = 65536) -> Iterator[bytes]:
+        path = self._path(self._storage_key(storage_ref))
+        if not path.exists():
+            raise NotFoundError("Blob not found", details={"storage_ref": storage_ref})
+        size = max(int(chunk_size), 1)
+        with path.open("rb") as handle:
+            while chunk := handle.read(size):
+                yield chunk
+
+    def stat(self, storage_ref: str) -> BlobRef:
+        storage_key = self._storage_key(storage_ref)
+        path = self._path(storage_key)
+        if not path.exists():
+            raise NotFoundError("Blob not found", details={"storage_ref": storage_ref})
+        return BlobRef(
+            digest=_digest_from_storage_key(storage_key),
+            size_bytes=path.stat().st_size,
+            storage_key=storage_key,
+            storage_ref=self._storage_ref(storage_key),
+            adapter=self.adapter_name,
+        )
+
+    def exists(self, storage_ref_or_digest: str) -> bool:
+        try:
+            storage_key = self._storage_key(storage_ref_or_digest)
+        except ValueError:
+            storage_key = digest_storage_key(storage_ref_or_digest)
+        return self._path(storage_key).exists()
+
+    def iter_blobs(self) -> list[BlobRef]:
+        root = self.root / "sha256"
+        if not root.exists():
+            return []
+        refs = []
+        for path in sorted(root.glob("*/*/*")):
+            if path.is_file() and not path.name.startswith("."):
+                storage_key = str(path.relative_to(self.root)).replace("\\", "/")
+                refs.append(self.stat(self._storage_ref(storage_key)))
+        return refs
+
+    def delete_unreferenced(
+        self,
+        referenced_storage_refs: set[str],
+        *,
+        dry_run: bool = True,
+    ) -> BlobCleanupResult:
+        referenced = {self._storage_key(ref) for ref in referenced_storage_refs if ref.startswith("blob://local/")}
+        deleted: list[str] = []
+        reclaimable = 0
+        retained = 0
+        for blob in self.iter_blobs():
+            if blob.storage_key in referenced:
+                retained += 1
+                continue
+            reclaimable += blob.size_bytes
+            deleted.append(blob.storage_ref)
+            if not dry_run:
+                self._path(blob.storage_key).unlink(missing_ok=True)
+        return BlobCleanupResult(
+            dry_run=dry_run,
+            deleted_count=len(deleted),
+            retained_count=retained,
+            reclaimable_bytes=reclaimable,
+            deleted_storage_refs=tuple(deleted),
+        )
+
+    def _path(self, storage_key: str) -> Path:
+        return self.root / storage_key
+
+    def _storage_ref(self, storage_key: str) -> str:
+        return f"blob://local/{storage_key}"
+
+    def _storage_key(self, storage_ref_or_digest: str) -> str:
+        if storage_ref_or_digest.startswith("blob://local/"):
+            return storage_ref_or_digest.removeprefix("blob://local/")
+        if storage_ref_or_digest.startswith("sha256:"):
+            return digest_storage_key(storage_ref_or_digest)
+        raise ValueError(f"Unsupported local blob reference: {storage_ref_or_digest}")
+
+
+def _digest_from_storage_key(storage_key: str) -> str:
+    return "sha256:" + storage_key.rsplit("/", 1)[-1]

src/kontextual_engine/adapters/memory/__init__.py

@@ -1,6 +1,6 @@
 """In-memory adapters for deterministic tests."""
 
 from .asset_registry import InMemoryAssetRegistryRepository
+from .blob_storage import InMemoryBlobStorage
 
-__all__ = ["InMemoryAssetRegistryRepository"]
-
+__all__ = ["InMemoryAssetRegistryRepository", "InMemoryBlobStorage"]

src/kontextual_engine/adapters/memory/blob_storage.py

@@ -0,0 +1,109 @@
+"""In-memory content-addressed blob storage for tests."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from kontextual_engine.errors import NotFoundError
+from kontextual_engine.ports import BlobCleanupResult, BlobRef, BlobWriteResult, blob_digest, digest_storage_key
+
+
+class InMemoryBlobStorage:
+    adapter_name = "memory"
+
+    def __init__(self) -> None:
+        self._blobs: dict[str, bytes] = {}
+        self._media_types: dict[str, str | None] = {}
+
+    def put_bytes(self, content: bytes, *, media_type: str | None = None) -> BlobWriteResult:
+        digest = blob_digest(content)
+        storage_key = digest_storage_key(digest)
+        storage_ref = self._storage_ref(storage_key)
+        created = storage_key not in self._blobs
+        if created:
+            self._blobs[storage_key] = bytes(content)
+            self._media_types[storage_key] = media_type
+        return BlobWriteResult(
+            BlobRef(
+                digest=digest,
+                size_bytes=len(content),
+                storage_key=storage_key,
+                storage_ref=storage_ref,
+                adapter=self.adapter_name,
+                media_type=media_type or self._media_types.get(storage_key),
+            ),
+            created=created,
+        )
+
+    def read_bytes(self, storage_ref: str) -> bytes:
+        storage_key = self._storage_key(storage_ref)
+        try:
+            return self._blobs[storage_key]
+        except KeyError as exc:
+            raise NotFoundError("Blob not found", details={"storage_ref": storage_ref}) from exc
+
+    def iter_bytes(self, storage_ref: str, *, chunk_size: int = 65536) -> Iterator[bytes]:
+        content = self.read_bytes(storage_ref)
+        size = max(int(chunk_size), 1)
+        for index in range(0, len(content), size):
+            yield content[index : index + size]
+
+    def stat(self, storage_ref: str) -> BlobRef:
+        content = self.read_bytes(storage_ref)
+        storage_key = self._storage_key(storage_ref)
+        return BlobRef(
+            digest=blob_digest(content),
+            size_bytes=len(content),
+            storage_key=storage_key,
+            storage_ref=self._storage_ref(storage_key),
+            adapter=self.adapter_name,
+            media_type=self._media_types.get(storage_key),
+        )
+
+    def exists(self, storage_ref_or_digest: str) -> bool:
+        try:
+            storage_key = self._storage_key(storage_ref_or_digest)
+        except ValueError:
+            storage_key = digest_storage_key(storage_ref_or_digest)
+        return storage_key in self._blobs
+
+    def iter_blobs(self) -> list[BlobRef]:
+        return [self.stat(self._storage_ref(storage_key)) for storage_key in sorted(self._blobs)]
+
+    def delete_unreferenced(
+        self,
+        referenced_storage_refs: set[str],
+        *,
+        dry_run: bool = True,
+    ) -> BlobCleanupResult:
+        referenced = {self._storage_key(ref) for ref in referenced_storage_refs if ref.startswith("blob://memory/")}
+        deleted: list[str] = []
+        reclaimable = 0
+        retained = 0
+        for storage_key, content in list(self._blobs.items()):
+            if storage_key in referenced:
+                retained += 1
+                continue
+            reclaimable += len(content)
+            storage_ref = self._storage_ref(storage_key)
+            deleted.append(storage_ref)
+            if not dry_run:
+                self._blobs.pop(storage_key, None)
+                self._media_types.pop(storage_key, None)
+        return BlobCleanupResult(
+            dry_run=dry_run,
+            deleted_count=len(deleted),
+            retained_count=retained,
+            reclaimable_bytes=reclaimable,
+            deleted_storage_refs=tuple(deleted),
+        )
+
+    def _storage_ref(self, storage_key: str) -> str:
+        return f"blob://memory/{storage_key}"
+
+    def _storage_key(self, storage_ref_or_digest: str) -> str:
+        if storage_ref_or_digest.startswith("blob://memory/"):
+            return storage_ref_or_digest.removeprefix("blob://memory/")
+        if storage_ref_or_digest.startswith("sha256:"):
+            return digest_storage_key(storage_ref_or_digest)
+        raise ValueError(f"Unsupported memory blob reference: {storage_ref_or_digest}")

src/kontextual_engine/adapters/s3/__init__.py

@@ -0,0 +1,6 @@
+"""S3-backed blob storage adapter."""
+
+from .blob_storage import S3BlobStorage
+
+__all__ = ["S3BlobStorage"]
+

src/kontextual_engine/adapters/s3/blob_storage.py

@@ -0,0 +1,198 @@
+"""S3 content-addressed blob storage adapter."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from kontextual_engine.errors import NotFoundError
+from kontextual_engine.ports import BlobCleanupResult, BlobRef, BlobWriteResult, blob_digest, digest_storage_key
+
+
+class S3BlobStorage:
+    adapter_name = "s3"
+
+    def __init__(
+        self,
+        *,
+        bucket: str,
+        prefix: str = "",
+        client: Any | None = None,
+    ) -> None:
+        self.bucket = bucket
+        self.prefix = prefix.strip("/")
+        if client is None:
+            import boto3  # type: ignore[import-not-found]
+
+            client = boto3.client("s3")
+        self.client = client
+
+    def put_bytes(self, content: bytes, *, media_type: str | None = None) -> BlobWriteResult:
+        digest = blob_digest(content)
+        storage_key = self._key(digest_storage_key(digest))
+        storage_ref = self._storage_ref(storage_key)
+        created = not self.exists(storage_ref)
+        if created:
+            kwargs: dict[str, Any] = {
+                "Bucket": self.bucket,
+                "Key": storage_key,
+                "Body": content,
+                "Metadata": {"digest": digest, "size-bytes": str(len(content))},
+            }
+            if media_type:
+                kwargs["ContentType"] = media_type
+            self.client.put_object(**kwargs)
+        return BlobWriteResult(
+            BlobRef(
+                digest=digest,
+                size_bytes=len(content),
+                storage_key=storage_key,
+                storage_ref=storage_ref,
+                adapter=self.adapter_name,
+                media_type=media_type,
+            ),
+            created=created,
+        )
+
+    def read_bytes(self, storage_ref: str) -> bytes:
+        storage_key = self._storage_key(storage_ref)
+        try:
+            result = self.client.get_object(Bucket=self.bucket, Key=storage_key)
+        except Exception as exc:
+            if _is_not_found(exc):
+                raise NotFoundError("Blob not found", details={"storage_ref": storage_ref}) from exc
+            raise
+        body = result["Body"]
+        return body.read() if hasattr(body, "read") else bytes(body)
+
+    def iter_bytes(self, storage_ref: str, *, chunk_size: int = 65536) -> Iterator[bytes]:
+        storage_key = self._storage_key(storage_ref)
+        try:
+            result = self.client.get_object(Bucket=self.bucket, Key=storage_key)
+        except Exception as exc:
+            if _is_not_found(exc):
+                raise NotFoundError("Blob not found", details={"storage_ref": storage_ref}) from exc
+            raise
+        body = result["Body"]
+        size = max(int(chunk_size), 1)
+        try:
+            if hasattr(body, "iter_chunks"):
+                for chunk in body.iter_chunks(chunk_size=size):
+                    if chunk:
+                        yield chunk
+                return
+            while True:
+                chunk = body.read(size) if hasattr(body, "read") else bytes(body)
+                if not chunk:
+                    break
+                yield chunk
+                if not hasattr(body, "read"):
+                    break
+        finally:
+            close = getattr(body, "close", None)
+            if close:
+                close()
+
+    def stat(self, storage_ref: str) -> BlobRef:
+        storage_key = self._storage_key(storage_ref)
+        try:
+            result = self.client.head_object(Bucket=self.bucket, Key=storage_key)
+        except Exception as exc:
+            if _is_not_found(exc):
+                raise NotFoundError("Blob not found", details={"storage_ref": storage_ref}) from exc
+            raise
+        metadata = dict(result.get("Metadata", {}))
+        digest = metadata.get("digest") or _digest_from_key(storage_key)
+        return BlobRef(
+            digest=digest,
+            size_bytes=int(result.get("ContentLength", metadata.get("size-bytes", 0))),
+            storage_key=storage_key,
+            storage_ref=self._storage_ref(storage_key),
+            adapter=self.adapter_name,
+            media_type=result.get("ContentType"),
+        )
+
+    def exists(self, storage_ref_or_digest: str) -> bool:
+        try:
+            self.stat(storage_ref_or_digest)
+            return True
+        except NotFoundError:
+            return False
+
+    def iter_blobs(self) -> list[BlobRef]:
+        prefix = f"{self.prefix}/sha256/" if self.prefix else "sha256/"
+        refs: list[BlobRef] = []
+        token: str | None = None
+        while True:
+            kwargs: dict[str, Any] = {"Bucket": self.bucket, "Prefix": prefix}
+            if token:
+                kwargs["ContinuationToken"] = token
+            result = self.client.list_objects_v2(**kwargs)
+            for item in result.get("Contents", []):
+                key = item["Key"]
+                refs.append(
+                    BlobRef(
+                        digest=_digest_from_key(key),
+                        size_bytes=int(item.get("Size", 0)),
+                        storage_key=key,
+                        storage_ref=self._storage_ref(key),
+                        adapter=self.adapter_name,
+                    )
+                )
+            if not result.get("IsTruncated"):
+                return refs
+            token = result.get("NextContinuationToken")
+
+    def delete_unreferenced(
+        self,
+        referenced_storage_refs: set[str],
+        *,
+        dry_run: bool = True,
+    ) -> BlobCleanupResult:
+        referenced = {self._storage_key(ref) for ref in referenced_storage_refs if ref.startswith(f"s3://{self.bucket}/")}
+        deleted: list[str] = []
+        reclaimable = 0
+        retained = 0
+        for blob in self.iter_blobs():
+            if blob.storage_key in referenced:
+                retained += 1
+                continue
+            deleted.append(blob.storage_ref)
+            reclaimable += blob.size_bytes
+            if not dry_run:
+                self.client.delete_object(Bucket=self.bucket, Key=blob.storage_key)
+        return BlobCleanupResult(
+            dry_run=dry_run,
+            deleted_count=len(deleted),
+            retained_count=retained,
+            reclaimable_bytes=reclaimable,
+            deleted_storage_refs=tuple(deleted),
+        )
+
+    def _key(self, storage_key: str) -> str:
+        return f"{self.prefix}/{storage_key}" if self.prefix else storage_key
+
+    def _storage_ref(self, storage_key: str) -> str:
+        return f"s3://{self.bucket}/{storage_key}"
+
+    def _storage_key(self, storage_ref_or_digest: str) -> str:
+        if storage_ref_or_digest.startswith(f"s3://{self.bucket}/"):
+            return storage_ref_or_digest.removeprefix(f"s3://{self.bucket}/")
+        if storage_ref_or_digest.startswith("sha256:"):
+            return self._key(digest_storage_key(storage_ref_or_digest))
+        if storage_ref_or_digest.startswith("blob://"):
+            raise ValueError(f"Unsupported S3 blob reference: {storage_ref_or_digest}")
+        return storage_ref_or_digest
+
+
+def _is_not_found(exc: Exception) -> bool:
+    response = getattr(exc, "response", None)
+    if isinstance(response, dict):
+        code = str(response.get("Error", {}).get("Code", ""))
+        status = str(response.get("ResponseMetadata", {}).get("HTTPStatusCode", ""))
+        return code in {"404", "NoSuchKey", "NotFound"} or status == "404"
+    return False
+
+
+def _digest_from_key(key: str) -> str:
+    return "sha256:" + key.rsplit("/", 1)[-1]