Tianyang_Gong/DocHub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add stdlib-only markdown and html text extraction for indexing

Browse Source
This commit is contained in:
Tianyang 2026-08-01 02:21:38 +08:00
parent f9c612927a
commit 2a7a197b95
2 changed files with 115 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
backend/app/text_extract.py Normal file
View File

@ -0,0 +1,38 @@
import re
from html.parser import HTMLParser
class _HTMLTextExtractor(HTMLParser):
def __init__(self):
super().__init__()
self._chunks = []
def handle_data(self, data):
self._chunks.append(data)
def get_text(self):
return "".join(self._chunks)
def html_to_text(text: str) -> str:
parser = _HTMLTextExtractor()
parser.feed(text)
parser.close()
raw = parser.get_text()
return re.sub(r"\s+", " ", raw).strip()
def markdown_to_text(text: str) -> str:
result = text
result = re.sub(r"```.*?```", " ", result, flags=re.DOTALL)
result = re.sub(r"`([^`]*)`", r"\1", result)
result = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", result)
result = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", result)
result = re.sub(r"^\s{0,3}#{1,6}\s*", "", result, flags=re.MULTILINE)
result = re.sub(r"(\*\*\*|___)(.+?)\1", r"\2", result)
result = re.sub(r"(\*\*|__)(.+?)\1", r"\2", result)
result = re.sub(r"(\*|_)(.+?)\1", r"\2", result)
result = re.sub(r"^\s*([-*+]|\d+\.)\s+", "", result, flags=re.MULTILINE)
result = re.sub(r"^\s*>\s*", "", result, flags=re.MULTILINE)
result = re.sub(r"\s+", " ", result).strip()
return result

77
backend/tests/test_text_extract.py Normal file
View File

@ -0,0 +1,77 @@
from app.text_extract import html_to_text, markdown_to_text
def test_html_to_text_strips_tags():
html = """
<html>
<body>
<h1>架构设计</h1>
<p>DocHub 是一个基于 <strong>MinIO</strong> 的文档管理系统。</p>
<ul>
<li>浏览文档</li>
<li>全文检索</li>
</ul>
</body>
</html>
"""
result = html_to_text(html)
assert "架构设计" in result
assert "DocHub 是一个基于 MinIO 的文档管理系统。" in result
assert "浏览文档" in result
assert "<h1>" not in result
assert "<strong>" not in result
def test_html_to_text_collapses_whitespace():
html = "<p>第一段</p>\n\n\n<p>第二段</p>"
result = html_to_text(html)
assert result == "第一段 第二段"
def test_markdown_to_text_strips_headers_and_emphasis():
markdown = """# 架构设计
## 背景
DocHub 是一个 **基于 MinIO** 的 _文档管理系统_，支持全文检索。
"""
result = markdown_to_text(markdown)
assert "#" not in result
assert "**" not in result
assert "_" not in result
assert "架构设计" in result
assert "背景" in result
assert "基于 MinIO" in result
assert "文档管理系统" in result
def test_markdown_to_text_strips_links_and_list_markers():
markdown = """## 功能列表
- 浏览 [MinIO](https://min.io) 中存储的文档
- 支持 [Meilisearch](https://www.meilisearch.com) 全文检索
1. 在线编辑并保存
"""
result = markdown_to_text(markdown)
assert "[" not in result
assert "](" not in result
assert "浏览 MinIO 中存储的文档" in result
assert "支持 Meilisearch 全文检索" in result
assert "在线编辑并保存" in result
def test_markdown_to_text_strips_code_fences_and_inline_code():
markdown = """使用 `pip install -r requirements.txt` 安装依赖。
```python
def hello():
return "world"
```
安装完成后启动服务。
"""
result = markdown_to_text(markdown)
assert "```" not in result
assert "def hello" not in result
assert "使用 pip install -r requirements.txt 安装依赖。" in result
assert "安装完成后启动服务。" in result