fix: natural-sort file tree, indent nested folders, fix editor layout
Also add a reindex endpoint and button to rebuild the Meilisearch index from files already on disk, without needing to re-upload them.
This commit is contained in:
parent
4a6fdaf994
commit
542361c63a
@ -1,11 +1,16 @@
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.text_extract import html_to_text, markdown_to_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _natural_sort_key(name: str) -> list:
|
||||
return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", name)]
|
||||
|
||||
ALLOWED_UPLOAD_EXTENSIONS = {".md", ".html", ".htm", ".png", ".jpg", ".jpeg", ".gif", ".webp"}
|
||||
MAX_UPLOAD_SIZE_BYTES = 10 * 1024 * 1024
|
||||
|
||||
@ -76,6 +81,8 @@ class DocumentService:
|
||||
"size": entry["size"],
|
||||
"modifiedAt": entry["modifiedAt"],
|
||||
})
|
||||
folders.sort(key=_natural_sort_key)
|
||||
files.sort(key=lambda f: _natural_sort_key(f["name"]))
|
||||
return {"folders": folders, "files": files}
|
||||
|
||||
def get_document(self, key: str) -> dict:
|
||||
@ -135,3 +142,24 @@ class DocumentService:
|
||||
self.save_document(key, data, content_type)
|
||||
name = key.rsplit("/", 1)[-1]
|
||||
return {"key": key, "name": name, "size": len(data)}
|
||||
|
||||
def reindex_all(self) -> dict:
|
||||
self._search_client.delete_all_documents()
|
||||
keys = self._file_client.walk_files()
|
||||
indexed = 0
|
||||
failed = 0
|
||||
for key in keys:
|
||||
content_type = _guess_content_type(key)
|
||||
if content_type == "application/octet-stream":
|
||||
continue
|
||||
try:
|
||||
data = self._file_client.get_object(key)
|
||||
text = self._extract_text(data, content_type)
|
||||
title = key.rsplit("/", 1)[-1]
|
||||
path = key.rsplit("/", 1)[0] if "/" in key else ""
|
||||
self._search_client.index_document(key, path, title, text)
|
||||
indexed += 1
|
||||
except Exception:
|
||||
logger.exception("Failed to reindex %s", key)
|
||||
failed += 1
|
||||
return {"indexed": indexed, "failed": failed, "total": len(keys)}
|
||||
|
||||
@ -74,3 +74,13 @@ class LocalFileClient:
|
||||
parent = parent.parent
|
||||
else:
|
||||
break
|
||||
|
||||
def walk_files(self) -> list[str]:
|
||||
"""递归遍历所有文件,返回相对于 base_dir 的 key 列表(不含目录)"""
|
||||
result = []
|
||||
for root, _dirs, filenames in os.walk(self._base_dir):
|
||||
for filename in filenames:
|
||||
full_path = Path(root) / filename
|
||||
rel_path = full_path.relative_to(self._base_dir)
|
||||
result.append(str(rel_path).replace(os.sep, "/"))
|
||||
return result
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.dependencies import get_search_client
|
||||
from app.dependencies import get_document_service, get_search_client
|
||||
from app.document_service import DocumentService
|
||||
from app.search_client import SearchClient
|
||||
|
||||
router = APIRouter()
|
||||
@ -9,3 +10,8 @@ router = APIRouter()
|
||||
@router.get("/api/search")
|
||||
def search(q: str = "", client: SearchClient = Depends(get_search_client)):
|
||||
return {"results": client.search(q)}
|
||||
|
||||
|
||||
@router.post("/api/search/reindex")
|
||||
def reindex(service: DocumentService = Depends(get_document_service)):
|
||||
return service.reindex_all()
|
||||
|
||||
@ -24,6 +24,10 @@ class SearchClient:
|
||||
index = self._client.index(self._index_name)
|
||||
index.delete_document(_encode_key(key))
|
||||
|
||||
def delete_all_documents(self) -> None:
|
||||
index = self._client.index(self._index_name)
|
||||
index.delete_all_documents()
|
||||
|
||||
def search(self, query: str) -> list[dict]:
|
||||
index = self._client.index(self._index_name)
|
||||
response = index.search(query, {
|
||||
|
||||
@ -35,6 +35,9 @@ class FakeFileClient:
|
||||
del self.stored[key]
|
||||
self.deleted_keys.append(key)
|
||||
|
||||
def walk_files(self):
|
||||
return list(self.stored.keys())
|
||||
|
||||
|
||||
class FakeSearchClient:
|
||||
def __init__(self, fail_on_index=False, fail_on_delete=False):
|
||||
@ -42,6 +45,7 @@ class FakeSearchClient:
|
||||
self.deleted = []
|
||||
self.fail_on_index = fail_on_index
|
||||
self.fail_on_delete = fail_on_delete
|
||||
self.cleared = False
|
||||
|
||||
def index_document(self, key, path, title, content):
|
||||
if self.fail_on_index:
|
||||
@ -53,6 +57,27 @@ class FakeSearchClient:
|
||||
raise RuntimeError("meilisearch unavailable")
|
||||
self.deleted.append(key)
|
||||
|
||||
def delete_all_documents(self):
|
||||
self.cleared = True
|
||||
|
||||
|
||||
def test_list_documents_sorts_files_naturally_by_numeric_prefix():
|
||||
file_client = FakeFileClient()
|
||||
file_client.entries = [
|
||||
{"key": "8.10 系统设计.md", "is_dir": False, "size": 1, "modifiedAt": None},
|
||||
{"key": "8.1 GMP调度.md", "is_dir": False, "size": 1, "modifiedAt": None},
|
||||
{"key": "8.2 Channel.md", "is_dir": False, "size": 1, "modifiedAt": None},
|
||||
]
|
||||
service = DocumentService(file_client, FakeSearchClient())
|
||||
|
||||
result = service.list_documents("")
|
||||
|
||||
assert [f["name"] for f in result["files"]] == [
|
||||
"8.1 GMP调度.md",
|
||||
"8.2 Channel.md",
|
||||
"8.10 系统设计.md",
|
||||
]
|
||||
|
||||
|
||||
def test_list_documents_at_root_returns_folders_and_files():
|
||||
file_client = FakeFileClient()
|
||||
@ -240,3 +265,28 @@ def test_upload_document_accepts_allowed_image_extension():
|
||||
|
||||
assert result == {"key": "产品文档/图片.png", "name": "图片.png", "size": len(data)}
|
||||
assert file_client.stored["产品文档/图片.png"] == data
|
||||
|
||||
|
||||
def test_reindex_all_clears_index_then_reindexes_every_supported_file():
|
||||
file_client = FakeFileClient()
|
||||
file_client.stored["产品文档/架构设计.md"] = "# 架构设计".encode("utf-8")
|
||||
file_client.stored["产品文档/图片.png"] = b"\x89PNG\r\n\x1a\n"
|
||||
search_client = FakeSearchClient()
|
||||
service = DocumentService(file_client, search_client)
|
||||
|
||||
result = service.reindex_all()
|
||||
|
||||
assert search_client.cleared
|
||||
assert result == {"indexed": 1, "failed": 0, "total": 2}
|
||||
assert search_client.indexed[0]["key"] == "产品文档/架构设计.md"
|
||||
|
||||
|
||||
def test_reindex_all_counts_failures_without_raising():
|
||||
file_client = FakeFileClient()
|
||||
file_client.stored["产品文档/架构设计.md"] = "# 架构设计".encode("utf-8")
|
||||
search_client = FakeSearchClient(fail_on_index=True)
|
||||
service = DocumentService(file_client, search_client)
|
||||
|
||||
result = service.reindex_all()
|
||||
|
||||
assert result == {"indexed": 0, "failed": 1, "total": 1}
|
||||
|
||||
@ -52,3 +52,21 @@ def test_list_objects_returns_empty_list_for_missing_prefix(client):
|
||||
def test_resolve_path_rejects_path_traversal(client):
|
||||
with pytest.raises(ValueError):
|
||||
client.get_object("../../etc/passwd")
|
||||
|
||||
|
||||
def test_walk_files_returns_all_nested_file_keys(client):
|
||||
client.put_object("产品文档/架构设计.md", b"# hi", "text/markdown")
|
||||
client.put_object("产品文档/子目录/详情.md", "# 详情".encode("utf-8"), "text/markdown")
|
||||
client.put_object("团队规范/编码规范.md", b"# rules", "text/markdown")
|
||||
|
||||
result = client.walk_files()
|
||||
|
||||
assert sorted(result) == sorted([
|
||||
"产品文档/架构设计.md",
|
||||
"产品文档/子目录/详情.md",
|
||||
"团队规范/编码规范.md",
|
||||
])
|
||||
|
||||
|
||||
def test_walk_files_returns_empty_list_when_no_files(client):
|
||||
assert client.walk_files() == []
|
||||
|
||||
@ -2,7 +2,7 @@ import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
from app.dependencies import get_search_client
|
||||
from app.dependencies import get_document_service, get_search_client
|
||||
|
||||
|
||||
class FakeSearchClient:
|
||||
@ -50,3 +50,29 @@ def test_search_returns_empty_results_for_no_match(client):
|
||||
def test_search_passes_query_string_to_search_client(client):
|
||||
client.get("/api/search", params={"q": "Garage"})
|
||||
assert client.fake_client.queries == ["Garage"]
|
||||
|
||||
|
||||
class FakeDocumentService:
|
||||
def __init__(self):
|
||||
self.reindex_called = False
|
||||
|
||||
def reindex_all(self):
|
||||
self.reindex_called = True
|
||||
return {"indexed": 3, "failed": 0, "total": 3}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reindex_client():
|
||||
fake_service = FakeDocumentService()
|
||||
app.dependency_overrides[get_document_service] = lambda: fake_service
|
||||
with TestClient(app) as test_client:
|
||||
test_client.fake_service = fake_service
|
||||
yield test_client
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_reindex_triggers_full_reindex_and_returns_summary(reindex_client):
|
||||
response = reindex_client.post("/api/search/reindex")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"indexed": 3, "failed": 0, "total": 3}
|
||||
assert reindex_client.fake_service.reindex_called
|
||||
|
||||
@ -4,6 +4,7 @@ import DocumentEditor from './components/DocumentEditor';
|
||||
import SearchBar from './components/SearchBar';
|
||||
import SearchResults from './components/SearchResults';
|
||||
import UploadButton from './components/UploadButton';
|
||||
import ReindexButton from './components/ReindexButton';
|
||||
import type { SearchResult, UploadDocumentResponse } from './api/client';
|
||||
|
||||
export default function App() {
|
||||
@ -66,6 +67,7 @@ export default function App() {
|
||||
DocHub <span className="subtitle">档案 · 检索</span>
|
||||
</h1>
|
||||
<SearchBar value={searchQuery} onQueryChange={handleQueryChange} onResults={handleResults} />
|
||||
<ReindexButton />
|
||||
<UploadButton path={uploadPath} onUploaded={handleUploaded} />
|
||||
</div>
|
||||
<div className="body">
|
||||
|
||||
@ -116,3 +116,17 @@ export async function search(query: string): Promise<SearchResponse> {
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export interface ReindexResponse {
|
||||
indexed: number;
|
||||
failed: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export async function reindexAll(): Promise<ReindexResponse> {
|
||||
const response = await fetch(`${API_BASE_URL}/api/search/reindex`, { method: 'POST' });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to reindex: ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
@ -121,7 +121,12 @@ export default function DocumentEditor({ documentKey, onDirtyChange, onDeleted }
|
||||
</div>
|
||||
</div>
|
||||
<div className="editor-area">
|
||||
<Editor value={content} onChange={handleChange} language={contentType === 'text/markdown' ? 'markdown' : 'html'} />
|
||||
<Editor
|
||||
value={content}
|
||||
onChange={handleChange}
|
||||
language={contentType === 'text/markdown' ? 'markdown' : 'html'}
|
||||
options={{ automaticLayout: true }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
41
frontend/src/components/ReindexButton.tsx
Normal file
41
frontend/src/components/ReindexButton.tsx
Normal file
@ -0,0 +1,41 @@
|
||||
import { useState } from 'react';
|
||||
import { reindexAll } from '../api/client';
|
||||
|
||||
export default function ReindexButton() {
|
||||
const [isRunning, setIsRunning] = useState(false);
|
||||
const [message, setMessage] = useState<string | null>(null);
|
||||
const [isError, setIsError] = useState(false);
|
||||
|
||||
function handleClick() {
|
||||
setIsRunning(true);
|
||||
setMessage(null);
|
||||
setIsError(false);
|
||||
reindexAll()
|
||||
.then((result) => {
|
||||
setMessage(`已重建索引 ${result.indexed}/${result.total}`);
|
||||
setIsError(result.failed > 0);
|
||||
setIsRunning(false);
|
||||
})
|
||||
.catch((err) => {
|
||||
setMessage(err instanceof Error ? err.message : '重建索引失败');
|
||||
setIsError(true);
|
||||
setIsRunning(false);
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="reindex-control">
|
||||
<button className="btn-secondary" onClick={handleClick} disabled={isRunning}>
|
||||
{isRunning ? '重建中…' : '重建索引'}
|
||||
</button>
|
||||
{message && (
|
||||
<span
|
||||
className={isError ? 'reindex-message error' : 'reindex-message'}
|
||||
data-testid="reindex-message"
|
||||
>
|
||||
{message}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -164,6 +164,23 @@ body {
|
||||
background: var(--pine-deep);
|
||||
}
|
||||
|
||||
.reindex-control {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.reindex-message {
|
||||
font-family: 'IBM Plex Mono', monospace;
|
||||
font-size: 11px;
|
||||
color: var(--ash);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.reindex-message.error {
|
||||
color: #b3432f;
|
||||
}
|
||||
|
||||
/* ---------- Body layout ---------- */
|
||||
.body {
|
||||
flex: 1;
|
||||
@ -233,6 +250,7 @@ body {
|
||||
|
||||
.tree-children {
|
||||
overflow: hidden;
|
||||
padding-left: 14px;
|
||||
}
|
||||
|
||||
.chevron {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { test, expect, vi, beforeEach } from 'vitest';
|
||||
import { listDocuments, getDocument, saveDocument, deleteDocument, uploadDocument, search } from '../../src/api/client';
|
||||
import { listDocuments, getDocument, saveDocument, deleteDocument, uploadDocument, search, reindexAll } from '../../src/api/client';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
@ -125,3 +125,22 @@ test('search calls GET /api/search with the q query param and returns parsed JSO
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(`/api/search?q=${encodeURIComponent('检索')}`);
|
||||
expect(result).toEqual(mockResponse);
|
||||
});
|
||||
|
||||
test('reindexAll posts to /api/search/reindex and returns parsed JSON', async () => {
|
||||
const mockResponse = { indexed: 5, failed: 0, total: 5 };
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve(mockResponse),
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const result = await reindexAll();
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith('/api/search/reindex', { method: 'POST' });
|
||||
expect(result).toEqual(mockResponse);
|
||||
});
|
||||
|
||||
test('reindexAll throws when the response is not ok', async () => {
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({ ok: false, status: 500 }) as unknown as typeof fetch;
|
||||
|
||||
await expect(reindexAll()).rejects.toThrow('Failed to reindex: 500');
|
||||
});
|
||||
|
||||
71
frontend/tests/components/ReindexButton.test.tsx
Normal file
71
frontend/tests/components/ReindexButton.test.tsx
Normal file
@ -0,0 +1,71 @@
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { vi, test, expect, beforeEach } from 'vitest';
|
||||
import ReindexButton from '../../src/components/ReindexButton';
|
||||
import { reindexAll } from '../../src/api/client';
|
||||
|
||||
vi.mock('../../src/api/client');
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(reindexAll).mockReset();
|
||||
});
|
||||
|
||||
test('clicking the button triggers reindexAll and shows the result summary', async () => {
|
||||
vi.mocked(reindexAll).mockResolvedValue({ indexed: 5, failed: 0, total: 5 });
|
||||
const user = userEvent.setup();
|
||||
|
||||
render(<ReindexButton />);
|
||||
await user.click(screen.getByRole('button', { name: '重建索引' }));
|
||||
|
||||
expect(reindexAll).toHaveBeenCalledTimes(1);
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('reindex-message')).toHaveTextContent('已重建索引 5/5');
|
||||
});
|
||||
});
|
||||
|
||||
test('shows an error style message when some documents fail to index', async () => {
|
||||
vi.mocked(reindexAll).mockResolvedValue({ indexed: 3, failed: 2, total: 5 });
|
||||
const user = userEvent.setup();
|
||||
|
||||
render(<ReindexButton />);
|
||||
await user.click(screen.getByRole('button', { name: '重建索引' }));
|
||||
|
||||
await waitFor(() => {
|
||||
const message = screen.getByTestId('reindex-message');
|
||||
expect(message).toHaveTextContent('已重建索引 3/5');
|
||||
expect(message).toHaveClass('error');
|
||||
});
|
||||
});
|
||||
|
||||
test('shows an error message when the request fails', async () => {
|
||||
vi.mocked(reindexAll).mockRejectedValue(new Error('Failed to reindex: 500'));
|
||||
const user = userEvent.setup();
|
||||
|
||||
render(<ReindexButton />);
|
||||
await user.click(screen.getByRole('button', { name: '重建索引' }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('reindex-message')).toHaveTextContent('Failed to reindex: 500');
|
||||
});
|
||||
});
|
||||
|
||||
test('disables the button while the request is in flight', async () => {
|
||||
let resolveFn: (value: { indexed: number; failed: number; total: number }) => void = () => {};
|
||||
vi.mocked(reindexAll).mockReturnValue(
|
||||
new Promise((resolve) => {
|
||||
resolveFn = resolve;
|
||||
}),
|
||||
);
|
||||
const user = userEvent.setup();
|
||||
|
||||
render(<ReindexButton />);
|
||||
const button = screen.getByRole('button', { name: '重建索引' });
|
||||
await user.click(button);
|
||||
|
||||
expect(screen.getByRole('button', { name: '重建中…' })).toBeDisabled();
|
||||
|
||||
resolveFn({ indexed: 1, failed: 0, total: 1 });
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole('button', { name: '重建索引' })).not.toBeDisabled();
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue
Block a user