add files

format_zalo.py

@@ -0,0 +1,120 @@
+"""Chuẩn hóa câu trả lời Hermes → text thuần cho Zalo (không markdown)."""
+from __future__ import annotations
+
+import re
+import unicodedata
+
+_MD_LINK = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
+_BOLD = re.compile(r"\*\*([^*]+)\*\*")
+_ITALIC = re.compile(r"(?<!\*)\*([^*]+)\*(?!\*)")
+_STRIKE = re.compile(r"~~([^~]+)~~")
+_CODE_FENCE = re.compile(r"```[\s\S]*?```")
+_INLINE_CODE = re.compile(r"`([^`]+)`")
+_HEADING = re.compile(r"(?m)^#{1,6}\s+")
+_BULLET = re.compile(r"(?m)^[\-\*]\s+")
+_ORDERED = re.compile(r"(?m)^\d+\.\s+")
+_BLOCKQUOTE = re.compile(r"(?m)^>\s?")
+_HR = re.compile(r"(?m)^(?:-{3,}|\*{3,}|_{3,})\s*$")
+_HTML_TAG = re.compile(r"</?[a-zA-Z][^>]*>")
+_WIKI_OR_REF = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|([^\]]+))?\]\]")
+_MULTI_NL = re.compile(r"\n{3,}")
+_MULTI_SPACE = re.compile(r"[ \t]{2,}")
+_DASHY = re.compile(r"\s*[—–]\s*")
+
+# Lỗi chính tả / khoảng trắng tiếng Việt thường gặp từ model
+_TYPO_FIXES: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"\s+([,.!?;:…])"), r"\1"),
+    (re.compile(r"([(\[])\s+"), r"\1"),
+    (re.compile(r"\s+([)\]])"), r"\1"),
+    (re.compile(r"(?i)\bko\b"), "không"),
+    (re.compile(r"(?i)\bdc\b"), "được"),
+    (re.compile(r"(?i)\bđc\b"), "được"),
+    (re.compile(r"(?i)\bkq\b"), "kết quả"),
+    (re.compile(r"(?i)\bntn\b"), "như thế nào"),
+    (re.compile(r"(?i)\bntn\b"), "như thế nào"),
+    (re.compile(r"(?i)\bmn\b"), "mọi người"),
+    (re.compile(r"(?i)\bbj\b"), "bây giờ"),
+    (re.compile(r"(?i)\bhqua\b"), "hôm qua"),
+    (re.compile(r"(?i)\bhnay\b"), "hôm nay"),
+    (re.compile(r"(?i)\bsdt\b"), "SĐT"),
+    (re.compile(r"(?i)\bvs\b"), "với"),
+    (re.compile(r"\u00a0"), " "),
+]
+
+
+def _nfc(text: str) -> str:
+    return unicodedata.normalize("NFC", text or "")
+
+
+def _strip_markdown(text: str) -> str:
+    t = text
+    t = _CODE_FENCE.sub(lambda m: "\n".join(
+        line for line in m.group(0).splitlines()[1:-1]
+    ), t)
+    t = _HTML_TAG.sub("", t)
+    t = _WIKI_OR_REF.sub(lambda m: (m.group(2) or m.group(1) or "").strip(), t)
+    t = _MD_LINK.sub(lambda m: m.group(1).strip() or m.group(2).strip(), t)
+    t = _BOLD.sub(r"\1", t)
+    t = _ITALIC.sub(r"\1", t)
+    t = _STRIKE.sub(r"\1", t)
+    t = _INLINE_CODE.sub(r"\1", t)
+    t = _HEADING.sub("", t)
+    t = _BLOCKQUOTE.sub("", t)
+    t = _HR.sub("", t)
+    t = _BULLET.sub("• ", t)
+    t = _ORDERED.sub(lambda m: f"{m.group(0).split('.')[0]}. ", t)
+    return t
+
+
+def _fix_vietnamese_typos(text: str) -> str:
+    t = text
+    for pat, repl in _TYPO_FIXES:
+        t = pat.sub(repl, t)
+    # khoảng trắng quanh dấu tiếng Việt kiểu "ạ ," / "không ?"
+    t = re.sub(r"([^\s])\s+([?!…])", r"\1\2", t)
+    return t
+
+
+def _tidy_lines(text: str) -> str:
+    lines: list[str] = []
+    for line in text.splitlines():
+        line = _MULTI_SPACE.sub(" ", line).strip()
+        if re.fullmatch(r"[—–\-.…\s]*", line or ""):
+            continue
+        if "—" in line or "–" in line:
+            parts = [p.strip() for p in _DASHY.split(line) if p.strip()]
+            if len(parts) > 1 and any(len(p) > 12 for p in parts):
+                lines.extend(parts)
+                continue
+            line = " ".join(parts)
+        if line:
+            lines.append(line)
+    t = "\n".join(lines)
+    t = _MULTI_NL.sub("\n\n", t).strip()
+    t = re.sub(r"\s+\.\s*$", ".", t)
+    t = re.sub(r"\.{2,}", "…", t)
+    t = re.sub(r"(?m)^[.\s]+$", "", t)
+    return _MULTI_NL.sub("\n\n", t).strip()
+
+
+def format_for_zalo(text: str, *, max_chars: int = 1800) -> str:
+    """Bỏ markdown, chuẩn hóa khoảng trắng / lỗi chính tả VN, cắt nếu quá dài."""
+    t = _nfc((text or "").strip())
+    if not t:
+        return t
+
+    t = _strip_markdown(t)
+    t = _fix_vietnamese_typos(t)
+    t = _tidy_lines(t)
+
+    if max_chars and len(t) > max_chars:
+        cut = t.rfind("\n", 0, max_chars)
+        if cut < max_chars // 3:
+            cut = t.rfind(". ", 0, max_chars)
+            if cut > 0:
+                cut += 1
+        if cut < max_chars // 3:
+            cut = max_chars
+        t = t[:cut].rstrip(" ,;:") + "…"
+
+    return t.strip()