diff --git a/backend/app/text_extract.py b/backend/app/text_extract.py new file mode 100644 index 0000000..7664ca8 --- /dev/null +++ b/backend/app/text_extract.py @@ -0,0 +1,38 @@ +import re +from html.parser import HTMLParser + + +class _HTMLTextExtractor(HTMLParser): + def __init__(self): + super().__init__() + self._chunks = [] + + def handle_data(self, data): + self._chunks.append(data) + + def get_text(self): + return "".join(self._chunks) + + +def html_to_text(text: str) -> str: + parser = _HTMLTextExtractor() + parser.feed(text) + parser.close() + raw = parser.get_text() + return re.sub(r"\s+", " ", raw).strip() + + +def markdown_to_text(text: str) -> str: + result = text + result = re.sub(r"```.*?```", " ", result, flags=re.DOTALL) + result = re.sub(r"`([^`]*)`", r"\1", result) + result = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", result) + result = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", result) + result = re.sub(r"^\s{0,3}#{1,6}\s*", "", result, flags=re.MULTILINE) + result = re.sub(r"(\*\*\*|___)(.+?)\1", r"\2", result) + result = re.sub(r"(\*\*|__)(.+?)\1", r"\2", result) + result = re.sub(r"(\*|_)(.+?)\1", r"\2", result) + result = re.sub(r"^\s*([-*+]|\d+\.)\s+", "", result, flags=re.MULTILINE) + result = re.sub(r"^\s*>\s*", "", result, flags=re.MULTILINE) + result = re.sub(r"\s+", " ", result).strip() + return result diff --git a/backend/tests/test_text_extract.py b/backend/tests/test_text_extract.py new file mode 100644 index 0000000..6c7cce6 --- /dev/null +++ b/backend/tests/test_text_extract.py @@ -0,0 +1,77 @@ +from app.text_extract import html_to_text, markdown_to_text + + +def test_html_to_text_strips_tags(): + html = """ + +
+DocHub 是一个基于 MinIO 的文档管理系统。
+第一段
\n\n\n第二段
" + result = html_to_text(html) + assert result == "第一段 第二段" + + +def test_markdown_to_text_strips_headers_and_emphasis(): + markdown = """# 架构设计 + +## 背景 + +DocHub 是一个 **基于 MinIO** 的 _文档管理系统_,支持全文检索。 +""" + result = markdown_to_text(markdown) + assert "#" not in result + assert "**" not in result + assert "_" not in result + assert "架构设计" in result + assert "背景" in result + assert "基于 MinIO" in result + assert "文档管理系统" in result + + +def test_markdown_to_text_strips_links_and_list_markers(): + markdown = """## 功能列表 + +- 浏览 [MinIO](https://min.io) 中存储的文档 +- 支持 [Meilisearch](https://www.meilisearch.com) 全文检索 +1. 在线编辑并保存 +""" + result = markdown_to_text(markdown) + assert "[" not in result + assert "](" not in result + assert "浏览 MinIO 中存储的文档" in result + assert "支持 Meilisearch 全文检索" in result + assert "在线编辑并保存" in result + + +def test_markdown_to_text_strips_code_fences_and_inline_code(): + markdown = """使用 `pip install -r requirements.txt` 安装依赖。 + +```python +def hello(): + return "world" +``` + +安装完成后启动服务。 +""" + result = markdown_to_text(markdown) + assert "```" not in result + assert "def hello" not in result + assert "使用 pip install -r requirements.txt 安装依赖。" in result + assert "安装完成后启动服务。" in result