39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
import re
|
|
from html.parser import HTMLParser
|
|
|
|
|
|
class _HTMLTextExtractor(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self._chunks = []
|
|
|
|
def handle_data(self, data):
|
|
self._chunks.append(data)
|
|
|
|
def get_text(self):
|
|
return "".join(self._chunks)
|
|
|
|
|
|
def html_to_text(text: str) -> str:
|
|
parser = _HTMLTextExtractor()
|
|
parser.feed(text)
|
|
parser.close()
|
|
raw = parser.get_text()
|
|
return re.sub(r"\s+", " ", raw).strip()
|
|
|
|
|
|
def markdown_to_text(text: str) -> str:
|
|
result = text
|
|
result = re.sub(r"```.*?```", " ", result, flags=re.DOTALL)
|
|
result = re.sub(r"`([^`]*)`", r"\1", result)
|
|
result = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", result)
|
|
result = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", result)
|
|
result = re.sub(r"^\s{0,3}#{1,6}\s*", "", result, flags=re.MULTILINE)
|
|
result = re.sub(r"(\*\*\*|___)(.+?)\1", r"\2", result)
|
|
result = re.sub(r"(\*\*|__)(.+?)\1", r"\2", result)
|
|
result = re.sub(r"(\*|_)(.+?)\1", r"\2", result)
|
|
result = re.sub(r"^\s*([-*+]|\d+\.)\s+", "", result, flags=re.MULTILINE)
|
|
result = re.sub(r"^\s*>\s*", "", result, flags=re.MULTILINE)
|
|
result = re.sub(r"\s+", " ", result).strip()
|
|
return result
|