feat: add stdlib-only markdown and html text extraction for indexing

2026-08-01 02:21:38 +08:00 · 2026-08-01 02:21:38 +08:00 · 2a7a197b95
commit 2a7a197b95
parent f9c612927a
2 changed files with 115 additions and 0 deletions
--- a/backend/app/text_extract.py
+++ b/backend/app/text_extract.py
@ -0,0 +1,38 @@
+import re
+from html.parser import HTMLParser
+
+
+class _HTMLTextExtractor(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self._chunks = []
+
+    def handle_data(self, data):
+        self._chunks.append(data)
+
+    def get_text(self):
+        return "".join(self._chunks)
+
+
+def html_to_text(text: str) -> str:
+    parser = _HTMLTextExtractor()
+    parser.feed(text)
+    parser.close()
+    raw = parser.get_text()
+    return re.sub(r"\s+", " ", raw).strip()
+
+
+def markdown_to_text(text: str) -> str:
+    result = text
+    result = re.sub(r"```.*?```", " ", result, flags=re.DOTALL)
+    result = re.sub(r"`([^`]*)`", r"\1", result)
+    result = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", result)
+    result = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", result)
+    result = re.sub(r"^\s{0,3}#{1,6}\s*", "", result, flags=re.MULTILINE)
+    result = re.sub(r"(\*\*\*|___)(.+?)\1", r"\2", result)
+    result = re.sub(r"(\*\*|__)(.+?)\1", r"\2", result)
+    result = re.sub(r"(\*|_)(.+?)\1", r"\2", result)
+    result = re.sub(r"^\s*([-*+]|\d+\.)\s+", "", result, flags=re.MULTILINE)
+    result = re.sub(r"^\s*>\s*", "", result, flags=re.MULTILINE)
+    result = re.sub(r"\s+", " ", result).strip()
+    return result
--- a/backend/tests/test_text_extract.py
+++ b/backend/tests/test_text_extract.py
@ -0,0 +1,77 @@
+from app.text_extract import html_to_text, markdown_to_text
+
+
+def test_html_to_text_strips_tags():
+    html = """
+    <html>
+      <body>
+        <h1>架构设计</h1>
+        <p>DocHub 是一个基于 <strong>MinIO</strong> 的文档管理系统。</p>
+        <ul>
+          <li>浏览文档</li>
+          <li>全文检索</li>
+        </ul>
+      </body>
+    </html>
+    """
+    result = html_to_text(html)
+    assert "架构设计" in result
+    assert "DocHub 是一个基于 MinIO 的文档管理系统。" in result
+    assert "浏览文档" in result
+    assert "<h1>" not in result
+    assert "<strong>" not in result
+
+
+def test_html_to_text_collapses_whitespace():
+    html = "<p>第一段</p>\n\n\n<p>第二段</p>"
+    result = html_to_text(html)
+    assert result == "第一段 第二段"
+
+
+def test_markdown_to_text_strips_headers_and_emphasis():
+    markdown = """# 架构设计
+
+## 背景
+
+DocHub 是一个 **基于 MinIO** 的 _文档管理系统_，支持全文检索。
+"""
+    result = markdown_to_text(markdown)
+    assert "#" not in result
+    assert "**" not in result
+    assert "_" not in result
+    assert "架构设计" in result
+    assert "背景" in result
+    assert "基于 MinIO" in result
+    assert "文档管理系统" in result
+
+
+def test_markdown_to_text_strips_links_and_list_markers():
+    markdown = """## 功能列表
+
+- 浏览 [MinIO](https://min.io) 中存储的文档
+- 支持 [Meilisearch](https://www.meilisearch.com) 全文检索
+1. 在线编辑并保存
+"""
+    result = markdown_to_text(markdown)
+    assert "[" not in result
+    assert "](" not in result
+    assert "浏览 MinIO 中存储的文档" in result
+    assert "支持 Meilisearch 全文检索" in result
+    assert "在线编辑并保存" in result
+
+
+def test_markdown_to_text_strips_code_fences_and_inline_code():
+    markdown = """使用 `pip install -r requirements.txt` 安装依赖。
+
+```python
+def hello():
+    return "world"
+```
+
+安装完成后启动服务。
+"""
+    result = markdown_to_text(markdown)
+    assert "```" not in result
+    assert "def hello" not in result
+    assert "使用 pip install -r requirements.txt 安装依赖。" in result
+    assert "安装完成后启动服务。" in result