refactor: rename all minio references to garage, drop minio SDK for boto3
The backend no longer mentions MinIO anywhere: MinioClient -> GarageClient (minio_client.py -> garage_client.py), MINIO_* env vars -> GARAGE_*, and the `minio` Python package is replaced with `boto3` (AWS's generic S3 client) since Garage is the only storage backend this project targets now. S3 API semantics are unchanged, so document_service.py's behavior is identical — this is a naming and dependency swap, not a logic change. Also updates test fixtures/names (FakeMinioClient -> FakeGarageClient, FakeMinioSDK -> FakeS3SDK) and storage/README.md, which no longer frames Garage as a MinIO replacement (that migration note has served its purpose now that no MinIO reference remains in the codebase). Backend: 54/54 tests passing. Frontend: 38/38 tests passing (unaffected, verified for regressions since it talks to the backend only through the unchanged REST API).
This commit is contained in:
parent
0f0b7e9d32
commit
360e9e5a14
@ -3,10 +3,10 @@ import os
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.minio_endpoint = os.environ.get("MINIO_ENDPOINT", "localhost:9000")
|
||||
self.minio_access_key = os.environ.get("MINIO_ACCESS_KEY", "minioadmin")
|
||||
self.minio_secret_key = os.environ.get("MINIO_SECRET_KEY", "minioadmin")
|
||||
self.minio_bucket = os.environ.get("MINIO_BUCKET", "dochub")
|
||||
self.garage_endpoint = os.environ.get("GARAGE_ENDPOINT", "localhost:3900")
|
||||
self.garage_access_key = os.environ.get("GARAGE_ACCESS_KEY", "")
|
||||
self.garage_secret_key = os.environ.get("GARAGE_SECRET_KEY", "")
|
||||
self.garage_bucket = os.environ.get("GARAGE_BUCKET", "dochub")
|
||||
self.meilisearch_host = os.environ.get("MEILISEARCH_HOST", "http://localhost:7700")
|
||||
self.meilisearch_api_key = os.environ.get("MEILISEARCH_API_KEY", "")
|
||||
self.meilisearch_index = os.environ.get("MEILISEARCH_INDEX", "documents")
|
||||
|
||||
@ -1,21 +1,21 @@
|
||||
from functools import lru_cache
|
||||
|
||||
import boto3
|
||||
import meilisearch
|
||||
from minio import Minio
|
||||
|
||||
from app.config import settings
|
||||
from app.document_service import DocumentService
|
||||
from app.minio_client import MinioClient
|
||||
from app.garage_client import GarageClient
|
||||
from app.search_client import SearchClient
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _get_minio_sdk() -> Minio:
|
||||
return Minio(
|
||||
settings.minio_endpoint,
|
||||
access_key=settings.minio_access_key,
|
||||
secret_key=settings.minio_secret_key,
|
||||
secure=False,
|
||||
def _get_garage_sdk():
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=f"http://{settings.garage_endpoint}",
|
||||
aws_access_key_id=settings.garage_access_key,
|
||||
aws_secret_access_key=settings.garage_secret_key,
|
||||
)
|
||||
|
||||
|
||||
@ -26,9 +26,9 @@ def _get_meilisearch_sdk() -> meilisearch.Client:
|
||||
|
||||
@lru_cache
|
||||
def get_document_service() -> DocumentService:
|
||||
minio_client = MinioClient(_get_minio_sdk(), settings.minio_bucket)
|
||||
garage_client = GarageClient(_get_garage_sdk(), settings.garage_bucket)
|
||||
search_client = SearchClient(_get_meilisearch_sdk(), settings.meilisearch_index)
|
||||
return DocumentService(minio_client, search_client)
|
||||
return DocumentService(garage_client, search_client)
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@ -53,13 +53,13 @@ def _guess_content_type(key: str) -> str:
|
||||
|
||||
|
||||
class DocumentService:
|
||||
def __init__(self, minio_client, search_client):
|
||||
self._minio_client = minio_client
|
||||
def __init__(self, garage_client, search_client):
|
||||
self._garage_client = garage_client
|
||||
self._search_client = search_client
|
||||
|
||||
def list_documents(self, path: str) -> dict:
|
||||
prefix = path if not path or path.endswith("/") else f"{path}/"
|
||||
entries = self._minio_client.list_objects(prefix)
|
||||
entries = self._garage_client.list_objects(prefix)
|
||||
folders = []
|
||||
files = []
|
||||
for entry in entries:
|
||||
@ -81,7 +81,7 @@ class DocumentService:
|
||||
def get_document(self, key: str) -> dict:
|
||||
_validate_path(key)
|
||||
try:
|
||||
data = self._minio_client.get_object(key)
|
||||
data = self._garage_client.get_object(key)
|
||||
except FileNotFoundError as exc:
|
||||
raise DocumentNotFoundError(key) from exc
|
||||
try:
|
||||
@ -107,7 +107,7 @@ class DocumentService:
|
||||
|
||||
def save_document(self, key: str, content: bytes, content_type: str) -> dict:
|
||||
_validate_path(key)
|
||||
self._minio_client.put_object(key, content, content_type)
|
||||
self._garage_client.put_object(key, content, content_type)
|
||||
try:
|
||||
text = self._extract_text(content, content_type)
|
||||
title = key.rsplit("/", 1)[-1]
|
||||
@ -119,7 +119,7 @@ class DocumentService:
|
||||
|
||||
def delete_document(self, key: str) -> None:
|
||||
_validate_path(key)
|
||||
self._minio_client.delete_object(key)
|
||||
self._garage_client.delete_object(key)
|
||||
try:
|
||||
self._search_client.delete_document(key)
|
||||
except Exception:
|
||||
|
||||
50
backend/app/garage_client.py
Normal file
50
backend/app/garage_client.py
Normal file
@ -0,0 +1,50 @@
|
||||
import io
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
class GarageClient:
|
||||
def __init__(self, client, bucket: str):
|
||||
self._client = client
|
||||
self._bucket = bucket
|
||||
|
||||
def list_objects(self, prefix: str) -> list[dict]:
|
||||
paginator = self._client.get_paginator("list_objects_v2")
|
||||
result = []
|
||||
for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix, Delimiter="/"):
|
||||
for common_prefix in page.get("CommonPrefixes", []):
|
||||
result.append({
|
||||
"key": common_prefix["Prefix"],
|
||||
"size": None,
|
||||
"is_dir": True,
|
||||
"modifiedAt": None,
|
||||
})
|
||||
for obj in page.get("Contents", []):
|
||||
result.append({
|
||||
"key": obj["Key"],
|
||||
"size": obj["Size"],
|
||||
"is_dir": obj["Key"].endswith("/"),
|
||||
"modifiedAt": obj["LastModified"].isoformat() if obj.get("LastModified") else None,
|
||||
})
|
||||
return result
|
||||
|
||||
def get_object(self, key: str) -> bytes:
|
||||
try:
|
||||
response = self._client.get_object(Bucket=self._bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
|
||||
raise FileNotFoundError(key) from exc
|
||||
raise
|
||||
return response["Body"].read()
|
||||
|
||||
def put_object(self, key: str, data: bytes, content_type: str) -> None:
|
||||
self._client.put_object(
|
||||
Bucket=self._bucket,
|
||||
Key=key,
|
||||
Body=io.BytesIO(data),
|
||||
ContentLength=len(data),
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def delete_object(self, key: str) -> None:
|
||||
self._client.delete_object(Bucket=self._bucket, Key=key)
|
||||
@ -4,7 +4,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.config import settings
|
||||
from app.dependencies import _get_minio_sdk
|
||||
from app.dependencies import _get_garage_sdk
|
||||
from app.routers import documents, search
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -26,11 +26,12 @@ app.include_router(search.router)
|
||||
@app.on_event("startup")
|
||||
def ensure_bucket_exists():
|
||||
try:
|
||||
minio_sdk = _get_minio_sdk()
|
||||
if not minio_sdk.bucket_exists(settings.minio_bucket):
|
||||
minio_sdk.make_bucket(settings.minio_bucket)
|
||||
garage_sdk = _get_garage_sdk()
|
||||
existing_buckets = {b["Name"] for b in garage_sdk.list_buckets().get("Buckets", [])}
|
||||
if settings.garage_bucket not in existing_buckets:
|
||||
garage_sdk.create_bucket(Bucket=settings.garage_bucket)
|
||||
except Exception:
|
||||
logger.warning("Could not verify/create MinIO bucket '%s' on startup", settings.minio_bucket, exc_info=True)
|
||||
logger.warning("Could not verify/create Garage bucket '%s' on startup", settings.garage_bucket, exc_info=True)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
|
||||
@ -1,45 +0,0 @@
|
||||
import io
|
||||
from minio.error import S3Error
|
||||
|
||||
|
||||
class MinioClient:
|
||||
def __init__(self, client, bucket: str):
|
||||
self._client = client
|
||||
self._bucket = bucket
|
||||
|
||||
def list_objects(self, prefix: str) -> list[dict]:
|
||||
objects = self._client.list_objects(self._bucket, prefix=prefix, recursive=False)
|
||||
result = []
|
||||
for obj in objects:
|
||||
result.append({
|
||||
"key": obj.object_name,
|
||||
"size": obj.size,
|
||||
"is_dir": obj.object_name.endswith("/"),
|
||||
"modifiedAt": obj.last_modified.isoformat() if obj.last_modified else None,
|
||||
})
|
||||
return result
|
||||
|
||||
def get_object(self, key: str) -> bytes:
|
||||
try:
|
||||
response = self._client.get_object(self._bucket, key)
|
||||
except S3Error as exc:
|
||||
if exc.code == "NoSuchKey":
|
||||
raise FileNotFoundError(key) from exc
|
||||
raise
|
||||
try:
|
||||
return response.read()
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
def put_object(self, key: str, data: bytes, content_type: str) -> None:
|
||||
self._client.put_object(
|
||||
self._bucket,
|
||||
key,
|
||||
io.BytesIO(data),
|
||||
length=len(data),
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
def delete_object(self, key: str) -> None:
|
||||
self._client.remove_object(self._bucket, key)
|
||||
@ -1,6 +1,6 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
minio==7.2.9
|
||||
boto3==1.35.36
|
||||
meilisearch==0.31.5
|
||||
pytest==8.3.3
|
||||
httpx==0.27.2
|
||||
|
||||
@ -10,7 +10,7 @@ from app.document_service import (
|
||||
)
|
||||
|
||||
|
||||
class FakeMinioClient:
|
||||
class FakeGarageClient:
|
||||
def __init__(self):
|
||||
self.entries = []
|
||||
self.stored = {}
|
||||
@ -55,13 +55,13 @@ class FakeSearchClient:
|
||||
|
||||
|
||||
def test_list_documents_at_root_returns_folders_and_files():
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.entries = [
|
||||
garage_client = FakeGarageClient()
|
||||
garage_client.entries = [
|
||||
{"key": "产品文档/", "is_dir": True, "size": 0, "modifiedAt": None},
|
||||
{"key": "团队规范/", "is_dir": True, "size": 0, "modifiedAt": None},
|
||||
{"key": "README.md", "is_dir": False, "size": 128, "modifiedAt": "2026-08-01T09:00:00Z"},
|
||||
]
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
result = service.list_documents("")
|
||||
|
||||
@ -72,9 +72,9 @@ def test_list_documents_at_root_returns_folders_and_files():
|
||||
|
||||
|
||||
def test_get_document_returns_content_and_content_type():
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.stored["产品文档/架构设计.md"] = "# 架构设计\n\nDocHub 使用 MinIO 存储文档。".encode("utf-8")
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
garage_client.stored["产品文档/架构设计.md"] = "# 架构设计\n\nDocHub 使用 Garage 存储文档。".encode("utf-8")
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
result = service.get_document("产品文档/架构设计.md")
|
||||
|
||||
@ -84,17 +84,17 @@ def test_get_document_returns_content_and_content_type():
|
||||
|
||||
|
||||
def test_get_document_raises_not_found_for_missing_key():
|
||||
service = DocumentService(FakeMinioClient(), FakeSearchClient())
|
||||
service = DocumentService(FakeGarageClient(), FakeSearchClient())
|
||||
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
service.get_document("不存在/文档.md")
|
||||
|
||||
|
||||
def test_get_document_returns_base64_content_for_binary_data_without_raising():
|
||||
minio_client = FakeMinioClient()
|
||||
garage_client = FakeGarageClient()
|
||||
binary_data = b"\x89PNG\r\n\x1a\n\x00\x01\xff\xfe"
|
||||
minio_client.stored["产品文档/图片.png"] = binary_data
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client.stored["产品文档/图片.png"] = binary_data
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
result = service.get_document("产品文档/图片.png")
|
||||
|
||||
@ -104,139 +104,139 @@ def test_get_document_returns_base64_content_for_binary_data_without_raising():
|
||||
|
||||
|
||||
def test_get_document_returns_utf8_encoding_for_text_content():
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.stored["产品文档/架构设计.md"] = "# 架构设计".encode("utf-8")
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
garage_client.stored["产品文档/架构设计.md"] = "# 架构设计".encode("utf-8")
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
result = service.get_document("产品文档/架构设计.md")
|
||||
|
||||
assert result["encoding"] == "utf-8"
|
||||
|
||||
|
||||
def test_save_document_writes_to_minio_then_indexes_content():
|
||||
minio_client = FakeMinioClient()
|
||||
def test_save_document_writes_to_garage_then_indexes_content():
|
||||
garage_client = FakeGarageClient()
|
||||
search_client = FakeSearchClient()
|
||||
service = DocumentService(minio_client, search_client)
|
||||
content = "# 架构设计\n\nDocHub 使用 **MinIO** 存储文档，使用 Meilisearch 做全文检索。".encode("utf-8")
|
||||
service = DocumentService(garage_client, search_client)
|
||||
content = "# 架构设计\n\nDocHub 使用 **Garage** 存储文档，使用 Meilisearch 做全文检索。".encode("utf-8")
|
||||
|
||||
result = service.save_document("产品文档/架构设计.md", content, "text/markdown")
|
||||
|
||||
assert minio_client.stored["产品文档/架构设计.md"] == content
|
||||
assert garage_client.stored["产品文档/架构设计.md"] == content
|
||||
assert search_client.indexed[0]["key"] == "产品文档/架构设计.md"
|
||||
assert search_client.indexed[0]["path"] == "产品文档"
|
||||
assert search_client.indexed[0]["title"] == "架构设计.md"
|
||||
assert "MinIO" in search_client.indexed[0]["content"]
|
||||
assert "Garage" in search_client.indexed[0]["content"]
|
||||
assert "#" not in search_client.indexed[0]["content"]
|
||||
assert "key" in result and "updatedAt" in result
|
||||
|
||||
|
||||
def test_save_document_does_not_fail_when_index_update_fails():
|
||||
minio_client = FakeMinioClient()
|
||||
garage_client = FakeGarageClient()
|
||||
search_client = FakeSearchClient(fail_on_index=True)
|
||||
service = DocumentService(minio_client, search_client)
|
||||
service = DocumentService(garage_client, search_client)
|
||||
content = b"# doc"
|
||||
|
||||
result = service.save_document("产品文档/架构设计.md", content, "text/markdown")
|
||||
|
||||
assert minio_client.stored["产品文档/架构设计.md"] == content
|
||||
assert garage_client.stored["产品文档/架构设计.md"] == content
|
||||
assert result["key"] == "产品文档/架构设计.md"
|
||||
|
||||
|
||||
def test_delete_document_removes_from_minio_and_search_index():
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.stored["产品文档/旧文档.md"] = b"old content"
|
||||
def test_delete_document_removes_from_garage_and_search_index():
|
||||
garage_client = FakeGarageClient()
|
||||
garage_client.stored["产品文档/旧文档.md"] = b"old content"
|
||||
search_client = FakeSearchClient()
|
||||
service = DocumentService(minio_client, search_client)
|
||||
service = DocumentService(garage_client, search_client)
|
||||
|
||||
service.delete_document("产品文档/旧文档.md")
|
||||
|
||||
assert "产品文档/旧文档.md" not in minio_client.stored
|
||||
assert "产品文档/旧文档.md" not in garage_client.stored
|
||||
assert search_client.deleted == ["产品文档/旧文档.md"]
|
||||
|
||||
|
||||
def test_delete_document_does_not_fail_when_index_delete_fails():
|
||||
minio_client = FakeMinioClient()
|
||||
minio_client.stored["产品文档/旧文档.md"] = b"old content"
|
||||
garage_client = FakeGarageClient()
|
||||
garage_client.stored["产品文档/旧文档.md"] = b"old content"
|
||||
search_client = FakeSearchClient(fail_on_delete=True)
|
||||
service = DocumentService(minio_client, search_client)
|
||||
service = DocumentService(garage_client, search_client)
|
||||
|
||||
service.delete_document("产品文档/旧文档.md")
|
||||
|
||||
assert "产品文档/旧文档.md" not in minio_client.stored
|
||||
assert "产品文档/旧文档.md" not in garage_client.stored
|
||||
|
||||
|
||||
def test_upload_document_writes_file_and_returns_metadata():
|
||||
minio_client = FakeMinioClient()
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
data = b"# uploaded doc"
|
||||
|
||||
result = service.upload_document("产品文档/新上传.md", data, "text/markdown")
|
||||
|
||||
assert result == {"key": "产品文档/新上传.md", "name": "新上传.md", "size": len(data)}
|
||||
assert minio_client.stored["产品文档/新上传.md"] == data
|
||||
assert garage_client.stored["产品文档/新上传.md"] == data
|
||||
|
||||
|
||||
def test_upload_document_rejects_unsupported_file_extension():
|
||||
minio_client = FakeMinioClient()
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
with pytest.raises(UnsupportedFileTypeError):
|
||||
service.upload_document("产品文档/病毒.exe", b"data", "application/octet-stream")
|
||||
|
||||
assert "产品文档/病毒.exe" not in minio_client.stored
|
||||
assert "产品文档/病毒.exe" not in garage_client.stored
|
||||
|
||||
|
||||
def test_upload_document_rejects_oversized_file():
|
||||
minio_client = FakeMinioClient()
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
data = b"x" * (MAX_UPLOAD_SIZE_BYTES + 1)
|
||||
|
||||
with pytest.raises(FileTooLargeError):
|
||||
service.upload_document("产品文档/大文件.md", data, "text/markdown")
|
||||
|
||||
assert "产品文档/大文件.md" not in minio_client.stored
|
||||
assert "产品文档/大文件.md" not in garage_client.stored
|
||||
|
||||
|
||||
def test_get_document_rejects_path_traversal_key():
|
||||
service = DocumentService(FakeMinioClient(), FakeSearchClient())
|
||||
service = DocumentService(FakeGarageClient(), FakeSearchClient())
|
||||
|
||||
with pytest.raises(InvalidPathError):
|
||||
service.get_document("../../etc/passwd")
|
||||
|
||||
|
||||
def test_save_document_rejects_path_traversal_key():
|
||||
minio_client = FakeMinioClient()
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
with pytest.raises(InvalidPathError):
|
||||
service.save_document("../../etc/passwd", b"data", "text/plain")
|
||||
|
||||
assert "../../etc/passwd" not in minio_client.stored
|
||||
assert "../../etc/passwd" not in garage_client.stored
|
||||
|
||||
|
||||
def test_delete_document_rejects_path_traversal_key():
|
||||
service = DocumentService(FakeMinioClient(), FakeSearchClient())
|
||||
service = DocumentService(FakeGarageClient(), FakeSearchClient())
|
||||
|
||||
with pytest.raises(InvalidPathError):
|
||||
service.delete_document("../../etc/passwd")
|
||||
|
||||
|
||||
def test_upload_document_rejects_path_traversal_key():
|
||||
minio_client = FakeMinioClient()
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
|
||||
with pytest.raises(InvalidPathError):
|
||||
service.upload_document("../../etc/passwd", b"data", "text/plain")
|
||||
|
||||
assert "../../etc/passwd" not in minio_client.stored
|
||||
assert "../../etc/passwd" not in garage_client.stored
|
||||
|
||||
|
||||
def test_upload_document_accepts_allowed_image_extension():
|
||||
minio_client = FakeMinioClient()
|
||||
service = DocumentService(minio_client, FakeSearchClient())
|
||||
garage_client = FakeGarageClient()
|
||||
service = DocumentService(garage_client, FakeSearchClient())
|
||||
data = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
result = service.upload_document("产品文档/图片.png", data, "image/png")
|
||||
|
||||
assert result == {"key": "产品文档/图片.png", "name": "图片.png", "size": len(data)}
|
||||
assert minio_client.stored["产品文档/图片.png"] == data
|
||||
assert garage_client.stored["产品文档/图片.png"] == data
|
||||
|
||||
100
backend/tests/test_garage_client.py
Normal file
100
backend/tests/test_garage_client.py
Normal file
@ -0,0 +1,100 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from app.garage_client import GarageClient
|
||||
|
||||
|
||||
class _FakePaginator:
|
||||
def __init__(self, sdk):
|
||||
self._sdk = sdk
|
||||
|
||||
def paginate(self, Bucket, Prefix="", Delimiter="/"):
|
||||
contents = []
|
||||
for key, (data, content_type, last_modified) in self._sdk.objects.items():
|
||||
if key.startswith(Prefix):
|
||||
contents.append({"Key": key, "Size": len(data), "LastModified": last_modified})
|
||||
yield {"Contents": contents, "CommonPrefixes": []}
|
||||
|
||||
|
||||
class FakeS3SDK:
|
||||
def __init__(self):
|
||||
self.objects = {}
|
||||
self.put_calls = []
|
||||
self.removed_keys = []
|
||||
|
||||
def get_paginator(self, name):
|
||||
return _FakePaginator(self)
|
||||
|
||||
def get_object(self, Bucket, Key):
|
||||
if Key not in self.objects:
|
||||
raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject")
|
||||
data, _, _ = self.objects[Key]
|
||||
|
||||
class _Body:
|
||||
def read(self_inner):
|
||||
return data
|
||||
|
||||
return {"Body": _Body()}
|
||||
|
||||
def put_object(self, Bucket, Key, Body, ContentLength, ContentType):
|
||||
data = Body.read()
|
||||
self.objects[Key] = (data, ContentType, datetime.now(timezone.utc))
|
||||
self.put_calls.append((Bucket, Key, ContentType))
|
||||
|
||||
def delete_object(self, Bucket, Key):
|
||||
del self.objects[Key]
|
||||
self.removed_keys.append(Key)
|
||||
|
||||
|
||||
def test_list_objects_returns_keys_matching_prefix():
|
||||
sdk = FakeS3SDK()
|
||||
sdk.objects["产品文档/架构设计.md"] = (b"# hi", "text/markdown", datetime(2026, 8, 1, tzinfo=timezone.utc))
|
||||
sdk.objects["团队规范/编码规范.md"] = (b"# rules", "text/markdown", datetime(2026, 8, 1, tzinfo=timezone.utc))
|
||||
client = GarageClient(sdk, bucket="dochub")
|
||||
|
||||
result = client.list_objects("产品文档/")
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["key"] == "产品文档/架构设计.md"
|
||||
assert result[0]["size"] == 4
|
||||
|
||||
|
||||
def test_get_object_returns_bytes():
|
||||
sdk = FakeS3SDK()
|
||||
sdk.objects["产品文档/架构设计.md"] = (b"# architecture", "text/markdown", datetime.now(timezone.utc))
|
||||
client = GarageClient(sdk, bucket="dochub")
|
||||
|
||||
data = client.get_object("产品文档/架构设计.md")
|
||||
|
||||
assert data == b"# architecture"
|
||||
|
||||
|
||||
def test_get_object_raises_file_not_found_for_missing_key():
|
||||
sdk = FakeS3SDK()
|
||||
client = GarageClient(sdk, bucket="dochub")
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
client.get_object("不存在/文档.md")
|
||||
|
||||
|
||||
def test_put_object_stores_data_with_content_type():
|
||||
sdk = FakeS3SDK()
|
||||
client = GarageClient(sdk, bucket="dochub")
|
||||
|
||||
client.put_object("产品文档/新文档.md", b"# new doc", "text/markdown")
|
||||
|
||||
assert sdk.objects["产品文档/新文档.md"][0] == b"# new doc"
|
||||
assert sdk.put_calls == [("dochub", "产品文档/新文档.md", "text/markdown")]
|
||||
|
||||
|
||||
def test_delete_object_removes_key():
|
||||
sdk = FakeS3SDK()
|
||||
sdk.objects["产品文档/旧文档.md"] = (b"old", "text/markdown", datetime.now(timezone.utc))
|
||||
client = GarageClient(sdk, bucket="dochub")
|
||||
|
||||
client.delete_object("产品文档/旧文档.md")
|
||||
|
||||
assert "产品文档/旧文档.md" not in sdk.objects
|
||||
assert sdk.removed_keys == ["产品文档/旧文档.md"]
|
||||
@ -18,33 +18,33 @@ def test_health_returns_ok():
|
||||
|
||||
|
||||
def test_startup_creates_bucket_when_missing(monkeypatch):
|
||||
fake_minio = MagicMock()
|
||||
fake_minio.bucket_exists.return_value = False
|
||||
monkeypatch.setattr(main_module, "_get_minio_sdk", lambda: fake_minio)
|
||||
fake_garage = MagicMock()
|
||||
fake_garage.list_buckets.return_value = {"Buckets": []}
|
||||
monkeypatch.setattr(main_module, "_get_garage_sdk", lambda: fake_garage)
|
||||
|
||||
with TestClient(app):
|
||||
pass
|
||||
|
||||
fake_minio.bucket_exists.assert_called()
|
||||
fake_minio.make_bucket.assert_called()
|
||||
fake_garage.list_buckets.assert_called()
|
||||
fake_garage.create_bucket.assert_called()
|
||||
|
||||
|
||||
def test_startup_does_not_create_bucket_when_it_already_exists(monkeypatch):
|
||||
fake_minio = MagicMock()
|
||||
fake_minio.bucket_exists.return_value = True
|
||||
monkeypatch.setattr(main_module, "_get_minio_sdk", lambda: fake_minio)
|
||||
fake_garage = MagicMock()
|
||||
fake_garage.list_buckets.return_value = {"Buckets": [{"Name": "dochub"}]}
|
||||
monkeypatch.setattr(main_module, "_get_garage_sdk", lambda: fake_garage)
|
||||
|
||||
with TestClient(app):
|
||||
pass
|
||||
|
||||
fake_minio.make_bucket.assert_not_called()
|
||||
fake_garage.create_bucket.assert_not_called()
|
||||
|
||||
|
||||
def test_startup_does_not_crash_app_when_minio_is_unreachable(monkeypatch):
|
||||
def test_startup_does_not_crash_app_when_garage_is_unreachable(monkeypatch):
|
||||
def raise_connection_error():
|
||||
raise ConnectionError("minio unreachable")
|
||||
raise ConnectionError("garage unreachable")
|
||||
|
||||
monkeypatch.setattr(main_module, "_get_minio_sdk", raise_connection_error)
|
||||
monkeypatch.setattr(main_module, "_get_garage_sdk", raise_connection_error)
|
||||
|
||||
with TestClient(app) as test_client:
|
||||
response = test_client.get("/health")
|
||||
|
||||
@ -1,108 +0,0 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from minio.error import S3Error
|
||||
|
||||
from app.minio_client import MinioClient
|
||||
|
||||
|
||||
class _FakeObject:
|
||||
def __init__(self, object_name, size, last_modified):
|
||||
self.object_name = object_name
|
||||
self.size = size
|
||||
self.last_modified = last_modified
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, data: bytes):
|
||||
self._data = data
|
||||
self.closed = False
|
||||
self.released = False
|
||||
|
||||
def read(self):
|
||||
return self._data
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
def release_conn(self):
|
||||
self.released = True
|
||||
|
||||
|
||||
class FakeMinioSDK:
|
||||
def __init__(self):
|
||||
self.objects = {}
|
||||
self.put_calls = []
|
||||
self.removed_keys = []
|
||||
|
||||
def list_objects(self, bucket, prefix="", recursive=False):
|
||||
for key, (data, content_type, last_modified) in self.objects.items():
|
||||
if key.startswith(prefix):
|
||||
yield _FakeObject(key, len(data), last_modified)
|
||||
|
||||
def get_object(self, bucket, key):
|
||||
if key not in self.objects:
|
||||
raise S3Error("NoSuchKey", "Object does not exist", key, "req-1", "host-1", None)
|
||||
data, _, _ = self.objects[key]
|
||||
return _FakeResponse(data)
|
||||
|
||||
def put_object(self, bucket, key, stream, length, content_type):
|
||||
data = stream.read()
|
||||
self.objects[key] = (data, content_type, datetime.now(timezone.utc))
|
||||
self.put_calls.append((bucket, key, content_type))
|
||||
|
||||
def remove_object(self, bucket, key):
|
||||
del self.objects[key]
|
||||
self.removed_keys.append(key)
|
||||
|
||||
|
||||
def test_list_objects_returns_keys_matching_prefix():
|
||||
sdk = FakeMinioSDK()
|
||||
sdk.objects["产品文档/架构设计.md"] = (b"# hi", "text/markdown", datetime(2026, 8, 1, tzinfo=timezone.utc))
|
||||
sdk.objects["团队规范/编码规范.md"] = (b"# rules", "text/markdown", datetime(2026, 8, 1, tzinfo=timezone.utc))
|
||||
client = MinioClient(sdk, bucket="dochub")
|
||||
|
||||
result = client.list_objects("产品文档/")
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["key"] == "产品文档/架构设计.md"
|
||||
assert result[0]["size"] == 4
|
||||
|
||||
|
||||
def test_get_object_returns_bytes():
|
||||
sdk = FakeMinioSDK()
|
||||
sdk.objects["产品文档/架构设计.md"] = (b"# architecture", "text/markdown", datetime.now(timezone.utc))
|
||||
client = MinioClient(sdk, bucket="dochub")
|
||||
|
||||
data = client.get_object("产品文档/架构设计.md")
|
||||
|
||||
assert data == b"# architecture"
|
||||
|
||||
|
||||
def test_get_object_raises_file_not_found_for_missing_key():
|
||||
sdk = FakeMinioSDK()
|
||||
client = MinioClient(sdk, bucket="dochub")
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
client.get_object("不存在/文档.md")
|
||||
|
||||
|
||||
def test_put_object_stores_data_with_content_type():
|
||||
sdk = FakeMinioSDK()
|
||||
client = MinioClient(sdk, bucket="dochub")
|
||||
|
||||
client.put_object("产品文档/新文档.md", b"# new doc", "text/markdown")
|
||||
|
||||
assert sdk.objects["产品文档/新文档.md"][0] == b"# new doc"
|
||||
assert sdk.put_calls == [("dochub", "产品文档/新文档.md", "text/markdown")]
|
||||
|
||||
|
||||
def test_delete_object_removes_key():
|
||||
sdk = FakeMinioSDK()
|
||||
sdk.objects["产品文档/旧文档.md"] = (b"old", "text/markdown", datetime.now(timezone.utc))
|
||||
client = MinioClient(sdk, bucket="dochub")
|
||||
|
||||
client.delete_object("产品文档/旧文档.md")
|
||||
|
||||
assert "产品文档/旧文档.md" not in sdk.objects
|
||||
assert sdk.removed_keys == ["产品文档/旧文档.md"]
|
||||
@ -48,5 +48,5 @@ def test_search_returns_empty_results_for_no_match(client):
|
||||
|
||||
|
||||
def test_search_passes_query_string_to_search_client(client):
|
||||
client.get("/api/search", params={"q": "MinIO"})
|
||||
assert client.fake_client.queries == ["MinIO"]
|
||||
client.get("/api/search", params={"q": "Garage"})
|
||||
assert client.fake_client.queries == ["Garage"]
|
||||
|
||||
@ -40,7 +40,7 @@ def test_index_document_adds_document_with_encoded_id():
|
||||
client = SearchClient(sdk, index_name="documents")
|
||||
|
||||
client.index_document(
|
||||
"产品文档/架构设计.md", "产品文档", "架构设计.md", "DocHub 使用 MinIO 存储文档"
|
||||
"产品文档/架构设计.md", "产品文档", "架构设计.md", "DocHub 使用 Garage 存储文档"
|
||||
)
|
||||
|
||||
index = sdk.indexes["documents"]
|
||||
@ -49,7 +49,7 @@ def test_index_document_adds_document_with_encoded_id():
|
||||
assert stored["key"] == "产品文档/架构设计.md"
|
||||
assert stored["path"] == "产品文档"
|
||||
assert stored["title"] == "架构设计.md"
|
||||
assert stored["content"] == "DocHub 使用 MinIO 存储文档"
|
||||
assert stored["content"] == "DocHub 使用 Garage 存储文档"
|
||||
|
||||
|
||||
def test_delete_document_removes_document():
|
||||
|
||||
@ -6,7 +6,7 @@ def test_html_to_text_strips_tags():
|
||||
<html>
|
||||
<body>
|
||||
<h1>架构设计</h1>
|
||||
<p>DocHub 是一个基于 <strong>MinIO</strong> 的文档管理系统。</p>
|
||||
<p>DocHub 是一个基于 <strong>Garage</strong> 的文档管理系统。</p>
|
||||
<ul>
|
||||
<li>浏览文档</li>
|
||||
<li>全文检索</li>
|
||||
@ -16,7 +16,7 @@ def test_html_to_text_strips_tags():
|
||||
"""
|
||||
result = html_to_text(html)
|
||||
assert "架构设计" in result
|
||||
assert "DocHub 是一个基于 MinIO 的文档管理系统。" in result
|
||||
assert "DocHub 是一个基于 Garage 的文档管理系统。" in result
|
||||
assert "浏览文档" in result
|
||||
assert "<h1>" not in result
|
||||
assert "<strong>" not in result
|
||||
@ -33,7 +33,7 @@ def test_markdown_to_text_strips_headers_and_emphasis():
|
||||
|
||||
## 背景
|
||||
|
||||
DocHub 是一个 **基于 MinIO** 的 _文档管理系统_，支持全文检索。
|
||||
DocHub 是一个 **基于 Garage** 的 _文档管理系统_，支持全文检索。
|
||||
"""
|
||||
result = markdown_to_text(markdown)
|
||||
assert "#" not in result
|
||||
@ -41,21 +41,21 @@ DocHub 是一个 **基于 MinIO** 的 _文档管理系统_，支持全文检索
|
||||
assert "_" not in result
|
||||
assert "架构设计" in result
|
||||
assert "背景" in result
|
||||
assert "基于 MinIO" in result
|
||||
assert "基于 Garage" in result
|
||||
assert "文档管理系统" in result
|
||||
|
||||
|
||||
def test_markdown_to_text_strips_links_and_list_markers():
|
||||
markdown = """## 功能列表
|
||||
|
||||
- 浏览 [MinIO](https://min.io) 中存储的文档
|
||||
- 浏览 [Garage](https://garagehq.deuxfleurs.fr) 中存储的文档
|
||||
- 支持 [Meilisearch](https://www.meilisearch.com) 全文检索
|
||||
1. 在线编辑并保存
|
||||
"""
|
||||
result = markdown_to_text(markdown)
|
||||
assert "[" not in result
|
||||
assert "](" not in result
|
||||
assert "浏览 MinIO 中存储的文档" in result
|
||||
assert "浏览 Garage 中存储的文档" in result
|
||||
assert "支持 Meilisearch 全文检索" in result
|
||||
assert "在线编辑并保存" in result
|
||||
|
||||
|
||||
@ -2,10 +2,10 @@ services:
|
||||
backend:
|
||||
build: ./backend
|
||||
environment:
|
||||
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
|
||||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
|
||||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
|
||||
MINIO_BUCKET: ${MINIO_BUCKET:-dochub}
|
||||
GARAGE_ENDPOINT: ${GARAGE_ENDPOINT:-garage:3900}
|
||||
GARAGE_ACCESS_KEY: ${GARAGE_ACCESS_KEY:-}
|
||||
GARAGE_SECRET_KEY: ${GARAGE_SECRET_KEY:-}
|
||||
GARAGE_BUCKET: ${GARAGE_BUCKET:-dochub}
|
||||
MEILISEARCH_HOST: http://meilisearch:7700
|
||||
MEILISEARCH_API_KEY: ${MEILISEARCH_API_KEY:-}
|
||||
MEILISEARCH_INDEX: ${MEILISEARCH_INDEX:-documents}
|
||||
|
||||
@ -1,13 +1,10 @@
|
||||
# DocHub Storage (Garage)
|
||||
|
||||
S3-compatible object storage for DocHub, deployed independently from the
|
||||
app stack in `docker-compose.yml` at the repo root — same separation as
|
||||
was used for MinIO, just backed by [Garage](https://garagehq.deuxfleurs.fr/)
|
||||
instead. Garage was chosen because MinIO's community edition has been
|
||||
scaling back features and its licensing direction has grown less
|
||||
predictable; Garage is fully open source (AGPL) and purpose-built for
|
||||
self-hosting. Both speak the S3 API, so `backend/app/minio_client.py`
|
||||
needed no code changes — only connection settings differ.
|
||||
app stack in `docker-compose.yml` at the repo root, using
|
||||
[Garage](https://garagehq.deuxfleurs.fr/) — a fully open source (AGPL)
|
||||
object store purpose-built for self-hosting. `backend/app/garage_client.py`
|
||||
talks to it over the standard S3 API via `boto3`.
|
||||
|
||||
Single-node setup, matching this project's learning/small-team scope
|
||||
(`replication_factor = 1` in `garage.toml` — no data redundancy across
|
||||
@ -23,20 +20,17 @@ docker compose exec garage /garage key info dochub-backend
|
||||
```
|
||||
|
||||
The last command prints an access key ID and secret key. Set those as
|
||||
`MINIO_ACCESS_KEY` / `MINIO_SECRET_KEY` for the DocHub backend (the env
|
||||
var names stay `MINIO_*` because `backend/app/config.py` was written
|
||||
against the S3 API, not against MinIO specifically).
|
||||
`GARAGE_ACCESS_KEY` / `GARAGE_SECRET_KEY` for the DocHub backend.
|
||||
|
||||
## Connecting the backend
|
||||
|
||||
From the repo root, when running `docker compose up` there:
|
||||
|
||||
```bash
|
||||
MINIO_ENDPOINT=<this-server-ip>:3900 \
|
||||
MINIO_ACCESS_KEY=<from init-garage.sh> \
|
||||
MINIO_SECRET_KEY=<from init-garage.sh> \
|
||||
GARAGE_ENDPOINT=<this-server-ip>:3900 \
|
||||
GARAGE_ACCESS_KEY=<from init-garage.sh> \
|
||||
GARAGE_SECRET_KEY=<from init-garage.sh> \
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Garage's S3 API listens on port `3900` (vs MinIO's `9000`) — that's the
|
||||
only endpoint difference the rest of the stack needs to know about.
|
||||
Garage's S3 API listens on port `3900`.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user