Tianyang
13789cc0b4
- main.py: remove duplicate /health endpoint definition - text_extract.py: HTML-escape extracted plain text before indexing so raw markup in uploaded documents can't render as live HTML via search snippets (stored XSS fix) - document_service.py: base64-encode content when UTF-8 decoding fails instead of raising UnicodeDecodeError (500) on binary files; add upload validation for file extension (allowlist) and size (10MB max) - routers/documents.py: map new validation errors to HTTP 400 with a clear detail message instead of letting them 500 - add/extend tests covering escaping, binary get_document, and upload validation rejection + acceptance paths
40 lines
1.2 KiB
Python
40 lines
1.2 KiB
Python
import html
|
|
import re
|
|
from html.parser import HTMLParser
|
|
|
|
|
|
class _HTMLTextExtractor(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self._chunks = []
|
|
|
|
def handle_data(self, data):
|
|
self._chunks.append(data)
|
|
|
|
def get_text(self):
|
|
return "".join(self._chunks)
|
|
|
|
|
|
def html_to_text(text: str) -> str:
|
|
parser = _HTMLTextExtractor()
|
|
parser.feed(text)
|
|
parser.close()
|
|
raw = parser.get_text()
|
|
return html.escape(re.sub(r"\s+", " ", raw).strip())
|
|
|
|
|
|
def markdown_to_text(text: str) -> str:
|
|
result = text
|
|
result = re.sub(r"```.*?```", " ", result, flags=re.DOTALL)
|
|
result = re.sub(r"`([^`]*)`", r"\1", result)
|
|
result = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", result)
|
|
result = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", result)
|
|
result = re.sub(r"^\s{0,3}#{1,6}\s*", "", result, flags=re.MULTILINE)
|
|
result = re.sub(r"(\*\*\*|___)(.+?)\1", r"\2", result)
|
|
result = re.sub(r"(\*\*|__)(.+?)\1", r"\2", result)
|
|
result = re.sub(r"(\*|_)(.+?)\1", r"\2", result)
|
|
result = re.sub(r"^\s*([-*+]|\d+\.)\s+", "", result, flags=re.MULTILINE)
|
|
result = re.sub(r"^\s*>\s*", "", result, flags=re.MULTILINE)
|
|
result = re.sub(r"\s+", " ", result).strip()
|
|
return html.escape(result)
|