refactor: replace Garage/S3 storage with local filesystem storage

Single-node deployment doesn't benefit from S3 abstraction overhead;
LocalFileClient keeps the same interface DocumentService depends on,
backed by a STORAGE_DIR volume mount instead of a separate Garage service.
This commit is contained in:
Tianyang 2026-08-01 16:30:02 +08:00
parent d2d046802b
commit 4c19fad561
12 changed files with 193 additions and 273 deletions

View File

@ -3,10 +3,7 @@ import os
class Settings:
def __init__(self):
self.garage_endpoint = os.environ.get("GARAGE_ENDPOINT", "localhost:3900")
self.garage_access_key = os.environ.get("GARAGE_ACCESS_KEY", "")
self.garage_secret_key = os.environ.get("GARAGE_SECRET_KEY", "")
self.garage_bucket = os.environ.get("GARAGE_BUCKET", "dochub")
self.storage_dir = os.environ.get("STORAGE_DIR", "/data/documents")
self.meilisearch_host = os.environ.get("MEILISEARCH_HOST", "http://localhost:7700")
self.meilisearch_api_key = os.environ.get("MEILISEARCH_API_KEY", "")
self.meilisearch_index = os.environ.get("MEILISEARCH_INDEX", "documents")

View File

@ -1,22 +1,16 @@
from functools import lru_cache
import boto3
import meilisearch
from app.config import settings
from app.document_service import DocumentService
from app.garage_client import GarageClient
from app.local_file_client import LocalFileClient
from app.search_client import SearchClient
@lru_cache
def _get_garage_sdk():
return boto3.client(
"s3",
endpoint_url=f"http://{settings.garage_endpoint}",
aws_access_key_id=settings.garage_access_key,
aws_secret_access_key=settings.garage_secret_key,
)
def _get_file_client() -> LocalFileClient:
return LocalFileClient(settings.storage_dir)
@lru_cache
@ -26,9 +20,9 @@ def _get_meilisearch_sdk() -> meilisearch.Client:
@lru_cache
def get_document_service() -> DocumentService:
garage_client = GarageClient(_get_garage_sdk(), settings.garage_bucket)
file_client = _get_file_client()
search_client = SearchClient(_get_meilisearch_sdk(), settings.meilisearch_index)
return DocumentService(garage_client, search_client)
return DocumentService(file_client, search_client)
@lru_cache

View File

@ -53,13 +53,13 @@ def _guess_content_type(key: str) -> str:
class DocumentService:
def __init__(self, garage_client, search_client):
self._garage_client = garage_client
def __init__(self, file_client, search_client):
self._file_client = file_client
self._search_client = search_client
def list_documents(self, path: str) -> dict:
prefix = path if not path or path.endswith("/") else f"{path}/"
entries = self._garage_client.list_objects(prefix)
entries = self._file_client.list_objects(prefix)
folders = []
files = []
for entry in entries:
@ -81,7 +81,7 @@ class DocumentService:
def get_document(self, key: str) -> dict:
_validate_path(key)
try:
data = self._garage_client.get_object(key)
data = self._file_client.get_object(key)
except FileNotFoundError as exc:
raise DocumentNotFoundError(key) from exc
try:
@ -107,7 +107,7 @@ class DocumentService:
def save_document(self, key: str, content: bytes, content_type: str) -> dict:
_validate_path(key)
self._garage_client.put_object(key, content, content_type)
self._file_client.put_object(key, content, content_type)
try:
text = self._extract_text(content, content_type)
title = key.rsplit("/", 1)[-1]
@ -119,7 +119,7 @@ class DocumentService:
def delete_document(self, key: str) -> None:
_validate_path(key)
self._garage_client.delete_object(key)
self._file_client.delete_object(key)
try:
self._search_client.delete_document(key)
except Exception:

View File

@ -1,50 +0,0 @@
import io
from botocore.exceptions import ClientError
class GarageClient:
def __init__(self, client, bucket: str):
self._client = client
self._bucket = bucket
def list_objects(self, prefix: str) -> list[dict]:
paginator = self._client.get_paginator("list_objects_v2")
result = []
for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix, Delimiter="/"):
for common_prefix in page.get("CommonPrefixes", []):
result.append({
"key": common_prefix["Prefix"],
"size": None,
"is_dir": True,
"modifiedAt": None,
})
for obj in page.get("Contents", []):
result.append({
"key": obj["Key"],
"size": obj["Size"],
"is_dir": obj["Key"].endswith("/"),
"modifiedAt": obj["LastModified"].isoformat() if obj.get("LastModified") else None,
})
return result
def get_object(self, key: str) -> bytes:
try:
response = self._client.get_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
raise FileNotFoundError(key) from exc
raise
return response["Body"].read()
def put_object(self, key: str, data: bytes, content_type: str) -> None:
self._client.put_object(
Bucket=self._bucket,
Key=key,
Body=io.BytesIO(data),
ContentLength=len(data),
ContentType=content_type,
)
def delete_object(self, key: str) -> None:
self._client.delete_object(Bucket=self._bucket, Key=key)

View File

@ -0,0 +1,76 @@
import os
from pathlib import Path
from datetime import datetime, timezone
class LocalFileClient:
"""本地文件系统存储客户端"""
def __init__(self, base_dir: str):
self._base_dir = Path(base_dir).resolve()
self._base_dir.mkdir(parents=True, exist_ok=True)
def _resolve_path(self, key: str) -> Path:
"""将 key 转换为文件系统路径,确保不会逃出 base_dir"""
path = (self._base_dir / key).resolve()
if not path.is_relative_to(self._base_dir):
raise ValueError(f"Path traversal attempt: {key}")
return path
def list_objects(self, prefix: str) -> list[dict]:
"""列出指定前缀下的所有对象(文件和目录)"""
prefix_path = self._base_dir / prefix if prefix else self._base_dir
if not prefix_path.exists():
return []
result = []
try:
for entry in prefix_path.iterdir():
rel_path = entry.relative_to(self._base_dir)
key = str(rel_path).replace(os.sep, "/")
if entry.is_dir():
result.append({
"key": key + "/",
"size": None,
"is_dir": True,
"modifiedAt": None,
})
else:
stat = entry.stat()
result.append({
"key": key,
"size": stat.st_size,
"is_dir": False,
"modifiedAt": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
})
except (PermissionError, OSError):
pass
return result
def get_object(self, key: str) -> bytes:
path = self._resolve_path(key)
if not path.exists() or not path.is_file():
raise FileNotFoundError(key)
return path.read_bytes()
def put_object(self, key: str, data: bytes, content_type: str) -> None:
path = self._resolve_path(key)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def delete_object(self, key: str) -> None:
path = self._resolve_path(key)
if not path.exists():
raise FileNotFoundError(key)
path.unlink()
# 清理空目录(逐层向上)
parent = path.parent
while parent != self._base_dir:
if not any(parent.iterdir()):
parent.rmdir()
parent = parent.parent
else:
break

View File

@ -4,7 +4,6 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.config import settings
from app.dependencies import _get_garage_sdk
from app.routers import documents, search
logger = logging.getLogger(__name__)
@ -23,17 +22,6 @@ app.include_router(documents.router)
app.include_router(search.router)
@app.on_event("startup")
def ensure_bucket_exists():
try:
garage_sdk = _get_garage_sdk()
existing_buckets = {b["Name"] for b in garage_sdk.list_buckets().get("Buckets", [])}
if settings.garage_bucket not in existing_buckets:
garage_sdk.create_bucket(Bucket=settings.garage_bucket)
except Exception:
logger.warning("Could not verify/create Garage bucket '%s' on startup", settings.garage_bucket, exc_info=True)
@app.get("/health")
def health():
return {"status": "ok"}

View File

@ -1,6 +1,5 @@
fastapi==0.115.0
uvicorn[standard]==0.30.6
boto3==1.35.36
meilisearch==0.31.5
pytest==8.3.3
httpx==0.27.2

View File

@ -10,7 +10,7 @@ from app.document_service import (
)
class FakeGarageClient:
class FakeFileClient:
def __init__(self):
self.entries = []
self.stored = {}
@ -55,13 +55,13 @@ class FakeSearchClient:
def test_list_documents_at_root_returns_folders_and_files():
garage_client = FakeGarageClient()
garage_client.entries = [
file_client = FakeFileClient()
file_client.entries = [
{"key": "产品文档/", "is_dir": True, "size": 0, "modifiedAt": None},
{"key": "团队规范/", "is_dir": True, "size": 0, "modifiedAt": None},
{"key": "README.md", "is_dir": False, "size": 128, "modifiedAt": "2026-08-01T09:00:00Z"},
]
service = DocumentService(garage_client, FakeSearchClient())
service = DocumentService(file_client, FakeSearchClient())
result = service.list_documents("")
@ -72,9 +72,9 @@ def test_list_documents_at_root_returns_folders_and_files():
def test_get_document_returns_content_and_content_type():
garage_client = FakeGarageClient()
garage_client.stored["产品文档/架构设计.md"] = "# 架构设计\n\nDocHub 使用 Garage 存储文档。".encode("utf-8")
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
file_client.stored["产品文档/架构设计.md"] = "# 架构设计\n\nDocHub 使用 Garage 存储文档。".encode("utf-8")
service = DocumentService(file_client, FakeSearchClient())
result = service.get_document("产品文档/架构设计.md")
@ -84,17 +84,17 @@ def test_get_document_returns_content_and_content_type():
def test_get_document_raises_not_found_for_missing_key():
service = DocumentService(FakeGarageClient(), FakeSearchClient())
service = DocumentService(FakeFileClient(), FakeSearchClient())
with pytest.raises(DocumentNotFoundError):
service.get_document("不存在/文档.md")
def test_get_document_returns_base64_content_for_binary_data_without_raising():
garage_client = FakeGarageClient()
file_client = FakeFileClient()
binary_data = b"\x89PNG\r\n\x1a\n\x00\x01\xff\xfe"
garage_client.stored["产品文档/图片.png"] = binary_data
service = DocumentService(garage_client, FakeSearchClient())
file_client.stored["产品文档/图片.png"] = binary_data
service = DocumentService(file_client, FakeSearchClient())
result = service.get_document("产品文档/图片.png")
@ -104,9 +104,9 @@ def test_get_document_returns_base64_content_for_binary_data_without_raising():
def test_get_document_returns_utf8_encoding_for_text_content():
garage_client = FakeGarageClient()
garage_client.stored["产品文档/架构设计.md"] = "# 架构设计".encode("utf-8")
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
file_client.stored["产品文档/架构设计.md"] = "# 架构设计".encode("utf-8")
service = DocumentService(file_client, FakeSearchClient())
result = service.get_document("产品文档/架构设计.md")
@ -114,14 +114,14 @@ def test_get_document_returns_utf8_encoding_for_text_content():
def test_save_document_writes_to_garage_then_indexes_content():
garage_client = FakeGarageClient()
file_client = FakeFileClient()
search_client = FakeSearchClient()
service = DocumentService(garage_client, search_client)
service = DocumentService(file_client, search_client)
content = "# 架构设计\n\nDocHub 使用 **Garage** 存储文档,使用 Meilisearch 做全文检索。".encode("utf-8")
result = service.save_document("产品文档/架构设计.md", content, "text/markdown")
assert garage_client.stored["产品文档/架构设计.md"] == content
assert file_client.stored["产品文档/架构设计.md"] == content
assert search_client.indexed[0]["key"] == "产品文档/架构设计.md"
assert search_client.indexed[0]["path"] == "产品文档"
assert search_client.indexed[0]["title"] == "架构设计.md"
@ -131,112 +131,112 @@ def test_save_document_writes_to_garage_then_indexes_content():
def test_save_document_does_not_fail_when_index_update_fails():
garage_client = FakeGarageClient()
file_client = FakeFileClient()
search_client = FakeSearchClient(fail_on_index=True)
service = DocumentService(garage_client, search_client)
service = DocumentService(file_client, search_client)
content = b"# doc"
result = service.save_document("产品文档/架构设计.md", content, "text/markdown")
assert garage_client.stored["产品文档/架构设计.md"] == content
assert file_client.stored["产品文档/架构设计.md"] == content
assert result["key"] == "产品文档/架构设计.md"
def test_delete_document_removes_from_garage_and_search_index():
garage_client = FakeGarageClient()
garage_client.stored["产品文档/旧文档.md"] = b"old content"
file_client = FakeFileClient()
file_client.stored["产品文档/旧文档.md"] = b"old content"
search_client = FakeSearchClient()
service = DocumentService(garage_client, search_client)
service = DocumentService(file_client, search_client)
service.delete_document("产品文档/旧文档.md")
assert "产品文档/旧文档.md" not in garage_client.stored
assert "产品文档/旧文档.md" not in file_client.stored
assert search_client.deleted == ["产品文档/旧文档.md"]
def test_delete_document_does_not_fail_when_index_delete_fails():
garage_client = FakeGarageClient()
garage_client.stored["产品文档/旧文档.md"] = b"old content"
file_client = FakeFileClient()
file_client.stored["产品文档/旧文档.md"] = b"old content"
search_client = FakeSearchClient(fail_on_delete=True)
service = DocumentService(garage_client, search_client)
service = DocumentService(file_client, search_client)
service.delete_document("产品文档/旧文档.md")
assert "产品文档/旧文档.md" not in garage_client.stored
assert "产品文档/旧文档.md" not in file_client.stored
def test_upload_document_writes_file_and_returns_metadata():
garage_client = FakeGarageClient()
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
service = DocumentService(file_client, FakeSearchClient())
data = b"# uploaded doc"
result = service.upload_document("产品文档/新上传.md", data, "text/markdown")
assert result == {"key": "产品文档/新上传.md", "name": "新上传.md", "size": len(data)}
assert garage_client.stored["产品文档/新上传.md"] == data
assert file_client.stored["产品文档/新上传.md"] == data
def test_upload_document_rejects_unsupported_file_extension():
garage_client = FakeGarageClient()
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
service = DocumentService(file_client, FakeSearchClient())
with pytest.raises(UnsupportedFileTypeError):
service.upload_document("产品文档/病毒.exe", b"data", "application/octet-stream")
assert "产品文档/病毒.exe" not in garage_client.stored
assert "产品文档/病毒.exe" not in file_client.stored
def test_upload_document_rejects_oversized_file():
garage_client = FakeGarageClient()
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
service = DocumentService(file_client, FakeSearchClient())
data = b"x" * (MAX_UPLOAD_SIZE_BYTES + 1)
with pytest.raises(FileTooLargeError):
service.upload_document("产品文档/大文件.md", data, "text/markdown")
assert "产品文档/大文件.md" not in garage_client.stored
assert "产品文档/大文件.md" not in file_client.stored
def test_get_document_rejects_path_traversal_key():
service = DocumentService(FakeGarageClient(), FakeSearchClient())
service = DocumentService(FakeFileClient(), FakeSearchClient())
with pytest.raises(InvalidPathError):
service.get_document("../../etc/passwd")
def test_save_document_rejects_path_traversal_key():
garage_client = FakeGarageClient()
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
service = DocumentService(file_client, FakeSearchClient())
with pytest.raises(InvalidPathError):
service.save_document("../../etc/passwd", b"data", "text/plain")
assert "../../etc/passwd" not in garage_client.stored
assert "../../etc/passwd" not in file_client.stored
def test_delete_document_rejects_path_traversal_key():
service = DocumentService(FakeGarageClient(), FakeSearchClient())
service = DocumentService(FakeFileClient(), FakeSearchClient())
with pytest.raises(InvalidPathError):
service.delete_document("../../etc/passwd")
def test_upload_document_rejects_path_traversal_key():
garage_client = FakeGarageClient()
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
service = DocumentService(file_client, FakeSearchClient())
with pytest.raises(InvalidPathError):
service.upload_document("../../etc/passwd", b"data", "text/plain")
assert "../../etc/passwd" not in garage_client.stored
assert "../../etc/passwd" not in file_client.stored
def test_upload_document_accepts_allowed_image_extension():
garage_client = FakeGarageClient()
service = DocumentService(garage_client, FakeSearchClient())
file_client = FakeFileClient()
service = DocumentService(file_client, FakeSearchClient())
data = b"\x89PNG\r\n\x1a\n"
result = service.upload_document("产品文档/图片.png", data, "image/png")
assert result == {"key": "产品文档/图片.png", "name": "图片.png", "size": len(data)}
assert garage_client.stored["产品文档/图片.png"] == data
assert file_client.stored["产品文档/图片.png"] == data

View File

@ -1,100 +0,0 @@
from datetime import datetime, timezone
import pytest
from botocore.exceptions import ClientError
from app.garage_client import GarageClient
class _FakePaginator:
def __init__(self, sdk):
self._sdk = sdk
def paginate(self, Bucket, Prefix="", Delimiter="/"):
contents = []
for key, (data, content_type, last_modified) in self._sdk.objects.items():
if key.startswith(Prefix):
contents.append({"Key": key, "Size": len(data), "LastModified": last_modified})
yield {"Contents": contents, "CommonPrefixes": []}
class FakeS3SDK:
def __init__(self):
self.objects = {}
self.put_calls = []
self.removed_keys = []
def get_paginator(self, name):
return _FakePaginator(self)
def get_object(self, Bucket, Key):
if Key not in self.objects:
raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject")
data, _, _ = self.objects[Key]
class _Body:
def read(self_inner):
return data
return {"Body": _Body()}
def put_object(self, Bucket, Key, Body, ContentLength, ContentType):
data = Body.read()
self.objects[Key] = (data, ContentType, datetime.now(timezone.utc))
self.put_calls.append((Bucket, Key, ContentType))
def delete_object(self, Bucket, Key):
del self.objects[Key]
self.removed_keys.append(Key)
def test_list_objects_returns_keys_matching_prefix():
sdk = FakeS3SDK()
sdk.objects["产品文档/架构设计.md"] = (b"# hi", "text/markdown", datetime(2026, 8, 1, tzinfo=timezone.utc))
sdk.objects["团队规范/编码规范.md"] = (b"# rules", "text/markdown", datetime(2026, 8, 1, tzinfo=timezone.utc))
client = GarageClient(sdk, bucket="dochub")
result = client.list_objects("产品文档/")
assert len(result) == 1
assert result[0]["key"] == "产品文档/架构设计.md"
assert result[0]["size"] == 4
def test_get_object_returns_bytes():
sdk = FakeS3SDK()
sdk.objects["产品文档/架构设计.md"] = (b"# architecture", "text/markdown", datetime.now(timezone.utc))
client = GarageClient(sdk, bucket="dochub")
data = client.get_object("产品文档/架构设计.md")
assert data == b"# architecture"
def test_get_object_raises_file_not_found_for_missing_key():
sdk = FakeS3SDK()
client = GarageClient(sdk, bucket="dochub")
with pytest.raises(FileNotFoundError):
client.get_object("不存在/文档.md")
def test_put_object_stores_data_with_content_type():
sdk = FakeS3SDK()
client = GarageClient(sdk, bucket="dochub")
client.put_object("产品文档/新文档.md", b"# new doc", "text/markdown")
assert sdk.objects["产品文档/新文档.md"][0] == b"# new doc"
assert sdk.put_calls == [("dochub", "产品文档/新文档.md", "text/markdown")]
def test_delete_object_removes_key():
sdk = FakeS3SDK()
sdk.objects["产品文档/旧文档.md"] = (b"old", "text/markdown", datetime.now(timezone.utc))
client = GarageClient(sdk, bucket="dochub")
client.delete_object("产品文档/旧文档.md")
assert "产品文档/旧文档.md" not in sdk.objects
assert sdk.removed_keys == ["产品文档/旧文档.md"]

View File

@ -0,0 +1,54 @@
import pytest
from app.local_file_client import LocalFileClient
@pytest.fixture
def client(tmp_path):
return LocalFileClient(str(tmp_path))
def test_put_and_get_object_roundtrip(client):
client.put_object("产品文档/架构设计.md", b"# hi", "text/markdown")
assert client.get_object("产品文档/架构设计.md") == b"# hi"
def test_get_object_raises_file_not_found_for_missing_key(client):
with pytest.raises(FileNotFoundError):
client.get_object("不存在/文档.md")
def test_delete_object_removes_file(client):
client.put_object("产品文档/旧文档.md", b"old", "text/markdown")
client.delete_object("产品文档/旧文档.md")
with pytest.raises(FileNotFoundError):
client.get_object("产品文档/旧文档.md")
def test_delete_object_raises_file_not_found_for_missing_key(client):
with pytest.raises(FileNotFoundError):
client.delete_object("不存在/文档.md")
def test_list_objects_returns_files_and_dirs_at_prefix(client):
client.put_object("产品文档/架构设计.md", b"# hi", "text/markdown")
client.put_object("团队规范/编码规范.md", b"# rules", "text/markdown")
result = client.list_objects("产品文档/")
assert len(result) == 1
assert result[0]["key"] == "产品文档/架构设计.md"
assert result[0]["size"] == 4
assert result[0]["is_dir"] is False
def test_list_objects_returns_empty_list_for_missing_prefix(client):
assert client.list_objects("不存在/") == []
def test_resolve_path_rejects_path_traversal(client):
with pytest.raises(ValueError):
client.get_object("../../etc/passwd")

View File

@ -1,9 +1,6 @@
from unittest.mock import MagicMock
import pytest
from fastapi.testclient import TestClient
import app.main as main_module
from app.main import app
from app.document_service import DocumentNotFoundError
from app.dependencies import get_document_service, get_search_client
@ -17,40 +14,6 @@ def test_health_returns_ok():
assert response.json() == {"status": "ok"}
def test_startup_creates_bucket_when_missing(monkeypatch):
fake_garage = MagicMock()
fake_garage.list_buckets.return_value = {"Buckets": []}
monkeypatch.setattr(main_module, "_get_garage_sdk", lambda: fake_garage)
with TestClient(app):
pass
fake_garage.list_buckets.assert_called()
fake_garage.create_bucket.assert_called()
def test_startup_does_not_create_bucket_when_it_already_exists(monkeypatch):
fake_garage = MagicMock()
fake_garage.list_buckets.return_value = {"Buckets": [{"Name": "dochub"}]}
monkeypatch.setattr(main_module, "_get_garage_sdk", lambda: fake_garage)
with TestClient(app):
pass
fake_garage.create_bucket.assert_not_called()
def test_startup_does_not_crash_app_when_garage_is_unreachable(monkeypatch):
def raise_connection_error():
raise ConnectionError("garage unreachable")
monkeypatch.setattr(main_module, "_get_garage_sdk", raise_connection_error)
with TestClient(app) as test_client:
response = test_client.get("/health")
assert response.status_code == 200
class FakeDocumentService:
def __init__(self):
self.storage = {}

View File

@ -2,14 +2,13 @@ services:
backend:
build: ./backend
environment:
GARAGE_ENDPOINT: ${GARAGE_ENDPOINT:-garage:3900}
GARAGE_ACCESS_KEY: ${GARAGE_ACCESS_KEY:-}
GARAGE_SECRET_KEY: ${GARAGE_SECRET_KEY:-}
GARAGE_BUCKET: ${GARAGE_BUCKET:-dochub}
STORAGE_DIR: /data/documents
MEILISEARCH_HOST: http://meilisearch:7700
MEILISEARCH_API_KEY: ${MEILISEARCH_API_KEY:-}
MEILISEARCH_INDEX: ${MEILISEARCH_INDEX:-documents}
CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:8080}
volumes:
- ${STORAGE_HOST_DIR:-../dochub-data}:/data/documents
depends_on:
- meilisearch
restart: unless-stopped