Update app.py

app.py

@@ -17,12 +17,12 @@ SAMPLE_JSON_MIN = """[
   },
   {
     "id": "ex-002",
-   "question": "question",
+    "question": "question",
     "answer": "answer"
   },
   {
     "id": "ex-003",
-   "question": "question",
+    "question": "question",
     "answer": "answer"
   }
 ]"""
@@ -33,6 +33,35 @@ def download_minimal_sample_json():
         tmp.flush()
         return tmp.name
 
+# ---------------------------
+# Helpers (formatting & menu-path)
+# ---------------------------
+def _normalize_ws(s: str) -> str:
+    return re.sub(r"\s+", " ", (s or "").strip())
+
+def _sentence_case(s: str) -> str:
+    s = _normalize_ws(s)
+    if not s:
+        return s
+    # single lowercase 'i' -> 'I'
+    s = re.sub(r"\bi\b", "I", s)
+    if s[0].islower():
+        s = s[0].upper() + s[1:]
+    if s[-1] not in ".!?":
+        s += "."
+    return s
+
+def _join_path(section: str | None, option: str | None) -> str | None:
+    section = (section or "").strip()
+    option = (option or "").strip()
+    if section and option:
+        return f"{section} > {option}"
+    if option:
+        return option
+    if section:
+        return section
+    return None
+
 # ---------------------------
 # Model: reward/quality (Transformers uyumlu)
 # ---------------------------
@@ -43,7 +72,7 @@ try:
         task="text-classification",
         model=MODEL_ID,
         tokenizer=MODEL_ID,
-        function_to_apply="none"  # regresyon skoru
+        function_to_apply="none"  # regression score
     )
     MODEL_READY = True
     LOAD_ERR = ""
@@ -55,7 +84,7 @@ except Exception as e:
 # Scoring & labeling
 # ---------------------------
 def score_pair(question: str, answer: str) -> float:
-    """Reward skoru (yüksek = daha kaliteli). Model yoksa yumuşak heuristik."""
+    """Reward score (higher = better). If model not ready, use light heuristic."""
     if not MODEL_READY:
         base = 0.3
         if question.strip().endswith("?"):
@@ -71,10 +100,10 @@ def score_pair(question: str, answer: str) -> float:
 
 def label_mapper_from_distribution(scores: List[float]):
     """
-    Skorlar negatif de olabilir; sabit eşikler yerine dağılım tabanlı eşik:
-      - low  : < 33. persentil
-      - medium: 33–66
-      - high : >= 66
+    Scores may be negative; use distribution-based thresholds:
+      low   : < 33rd percentile
+      medium: 33–66
+      high  : >= 66
     """
     if not scores:
         return lambda s: "medium"
@@ -100,8 +129,7 @@ def label_mapper_from_distribution(scores: List[float]):
 # ---------------------------
 def _extract_module(item: Dict[str, Any], q_text: str) -> str | None:
     """
-    'CUSTOMERS' gibi modül adını soru+context içinden yakalar.
-    Büyük harfli 3+ harfli kelimeleri modül adayı sayıyoruz.
+    Extract uppercase 3+ letter 'module-like' token from question/context, e.g., CUSTOMERS.
     """
     ctx = f"{item.get('context','')} {q_text}"
     m = re.search(r"\b([A-Z]{3,})\b", ctx)
@@ -109,10 +137,9 @@ def _extract_module(item: Dict[str, Any], q_text: str) -> str | None:
 
 def _roles_from_answer(ans: str) -> List[str]:
     """
-    Cevaptan rol listesini çıkarır; Title Case yapar; boşları atar.
-    Örn: "Sales representatives, customer consultants and sales managers"
+    Pull a role list from the answer; Title Case; drop empties.
     """
-    parts = re.split(r",| and ", ans, flags=re.IGNORECASE)
+    parts = re.split(r",| and ", ans or "", flags=re.IGNORECASE)
     roles = []
     for p in parts:
         t = p.strip(" .")
@@ -122,33 +149,31 @@ def _roles_from_answer(ans: str) -> List[str]:
         roles.append(t)
     return [r for r in roles if r]
 
-def _std_sentence(s: str) -> str:
-    s = s.strip()
-    if not s:
-        return s
-    if s[0].islower():
-        s = s[0].upper() + s[1:]
-    if s[-1] not in ".!?":
-        s += "."
-    return s
-
+# ---------------------------
+# Rewriter (WHO/WHERE/WHAT/HOW aware)
+# ---------------------------
 def improve_smart(item: Dict[str, Any]) -> Dict[str, Any]:
     """
-    TOPLAM DÜZENLEYİCİ:
-    - WHO tipi ve modül adı varsa: "Which roles are authorized to access the {MODULE} module in DealerTIQ?"
-      Cevap: "Authorized roles include A, B, and C."
-    - Aksi halde genel temizlik (soru işareti, büyük harf, noktalama, çok kısa cevabı kısaca netleştirme).
+    LLM-free safe rewrite:
+    - WHO + MODULE: use roles pattern. (Only here!)
+    - WHERE: produce menu-path sentence (e.g., Settings > Inventory Parameters).
+    - WHAT: definition/purpose or 'allows searching by …' sentence.
+    - HOW: short procedural sentence.
+    - Else: grammar/format normalization.
     """
-    q = (item.get("question") or "").strip()
-    a = (item.get("answer") or "").strip()
+    q = _normalize_ws(item.get("question") or "")
+    a = _normalize_ws(item.get("answer") or "")
     meta = item.get("metadata") or {}
     qtype = (meta.get("question_type") or "").lower()
+    orig_q = _normalize_ws(item.get("original_question") or "")
+    orig_a = _normalize_ws(item.get("original_answer") or "")
+    base = orig_a or a  # prefer original to keep semantics
 
-    module = _extract_module(item, q)
-    roles = _roles_from_answer(a)
+    module = _extract_module(item, q)  # e.g., CUSTOMERS
 
-    # WHO tipi + modül saptandıysa güçlü şablon
+    # -- WHO: ONLY here we use roles pattern
     if qtype == "who" and module:
+        roles = _roles_from_answer(base)
         new_q = f"Which roles are authorized to access the {module} module in DealerTIQ?"
         if roles:
             if len(roles) == 1:
@@ -159,48 +184,81 @@ def improve_smart(item: Dict[str, Any]) -> Dict[str, Any]:
                 roles_str = ", ".join(roles[:-1]) + f", and {roles[-1]}"
             new_a = f"Authorized roles include {roles_str}."
         else:
-            # Rol çıkarılamadıysa mevcut cevabı sadece standardize et
-            new_a = _std_sentence(a) if a else a
+            new_a = _sentence_case(base) if base else _sentence_case(a)
 
-        # context'i de standardize et (varsa)
         if item.get("context"):
             item["context"] = f"DealerTIQ — {module} module"
 
-        item["question"] = new_q
-        item["answer"] = new_a
+        item["question"] = _sentence_case(new_q[:-1] + "?")
+        item["answer"] = _sentence_case(new_a)
         return item
 
-    # Genel temizlik (LLM yok; güvenli kurallar)
-    if q:
-        q2 = q
-        # "permission to use X" → "authorized to access the X module"
-        m = re.search(r"permission to use\s+([A-Z]{3,})", q2, flags=re.IGNORECASE)
-        if m:
-            mod = m.group(1)
-            q2 = f"Which roles are authorized to access the {mod} module in DealerTIQ"
-        if not q2.endswith("?"):
-            q2 += "?"
-        if q2[0].islower():
-            q2 = q2[0].upper() + q2[1:]
-        q = q2
-
-    if a:
-        a2 = a.strip()
-        if roles:
-            if len(roles) == 1:
-                roles_str = roles[0]
-            elif len(roles) == 2:
-                roles_str = " and ".join(roles)
+    # -- WHERE: menu path sentence
+    if qtype == "where":
+        text = base or a
+        # "under the Settings section"
+        m_sec = re.search(r"under the\s+([A-Za-z ]+?)\s+section", text or "", flags=re.IGNORECASE)
+        section = m_sec.group(1).strip().title() if m_sec else None
+        # quoted option: "Inventory Parameters"
+        quotes = re.findall(r'"([^"]+)"', text or "")
+        option = quotes[0].strip() if quotes else None
+
+        path = _join_path(section, option)
+        target = option or (module and f"{module} module") or "page"
+        new_q = f"Where is the {target} located in DealerTIQ?"
+        new_a = f"It is located under {path} in the left navigation menu." if path else (text or "It is available in the left navigation menu.")
+
+        item["question"] = _sentence_case(new_q[:-1] + "?")
+        item["answer"] = _sentence_case(new_a)
+        return item
+
+    # -- WHAT: definition/purpose or 'allows searching by …'
+    if qtype == "what":
+        text = base or a
+        if re.search(r"allows\s+search(ing)?\s+by", text or "", flags=re.IGNORECASE):
+            m = re.search(r"such as\s+(.+)", text or "", flags=re.IGNORECASE)
+            if m:
+                feats = m.group(1).strip().rstrip(".")
+                new_q = f"What can you search for in {module or 'this module'}?"
+                new_a = f"It allows searching by criteria such as {feats}."
             else:
-                roles_str = ", ".join(roles[:-1]) + f", and {roles[-1]}"
-            a2 = f"Authorized roles include {roles_str}."
-        a = _std_sentence(a2)
+                new_q = orig_q or q or "What can you search for in this module?"
+                new_a = text or "It allows searching by multiple criteria."
+        else:
+            if re.search(r"\b(configure|configuration|settings)\b", text or "", flags=re.IGNORECASE):
+                target = module or "Inventory Parameters"
+                new_q = f"What is configured in the {target}?"
+                new_a = text or "It configures related settings and rules."
+            else:
+                new_q = orig_q or q or "What is the purpose of this module?"
+                new_a = text or "It provides the core functionality for this area."
+        item["question"] = _sentence_case(new_q[:-1] + "?")
+        item["answer"] = _sentence_case(new_a)
+        return item
 
-    if a and len(a.split()) < 6:
-        a = a + " This answer has been clarified for brevity and precision."
+    # -- HOW: short procedure
+    if qtype == "how":
+        text = base or a
+        quotes = re.findall(r'"([^"]+)"', text or "")  # "Add Channel", "Inventory Parameters"
+        m_sec = re.search(r"under the\s+([A-Za-z ]+?)\s+section", text or "", flags=re.IGNORECASE)
+        section = m_sec.group(1).strip().title() if m_sec else None
+        path = _join_path(section, quotes[0] if quotes else None)
+
+        new_q = orig_q or q or f"How do I perform this action in {module or 'the module'}?"
+        steps = _normalize_ws(text or "")
+        steps = re.sub(r"\bclick on\b", "select", steps, flags=re.IGNORECASE)
+        if path and "left navigation" not in steps.lower():
+            steps = f"Go to {path} in the left navigation menu, then {steps[0].lower() + steps[1:]}" if steps else f"Go to {path} in the left navigation menu."
+
+        item["question"] = _sentence_case(new_q[:-1] + "?") if not new_q.endswith("?") else _sentence_case(new_q)
+        item["answer"] = _sentence_case(steps or "Follow the on-screen instructions to complete the action.")
+        return item
 
-    item["question"] = q
-    item["answer"] = a
+    # -- Fallback: grammar/format normalization only
+    if q and not q.endswith("?"):
+        q += "?"
+    item["question"] = _sentence_case(q)
+    item["answer"] = _sentence_case(base or a)
     return item
 
 # ---------------------------
@@ -208,22 +266,22 @@ def improve_smart(item: Dict[str, Any]) -> Dict[str, Any]:
 # ---------------------------
 def process_json(file) -> Tuple[List[Dict[str, Any]], str, str, str]:
     """
-    Girdi: JSON (list veya tek obje)
-    Adımlar:
-      1) İlk skor turu (tüm örnekler)
-      2) Dağılım tabanlı label (low/medium/high)
-      3) LOW olanları otomatik rewrite (improve_smart)
-      4) Yeniden skorla, quality_before / quality_after alanlarını yaz
-    Çıktı:
+    Input: JSON (list or single object)
+    Steps:
+      1) First scoring pass for all items
+      2) Label via distribution thresholds (low/medium/high)
+      3) Auto-rewrite items labeled 'low' (improve_smart)
+      4) Rescore & write quality_before / quality_after
+    Output:
       - Summary (Dataframe)
-      - Preview JSON (ilk 50)
+      - Preview JSON (first 50)
       - Download JSON path
       - Warn/Info
     """
     data = json.load(open(file.name))
     items: List[Dict[str, Any]] = data if isinstance(data, list) else [data]
 
-    # 1) İlk skor turu
+    # 1) First scoring pass
     first = []
     scores = []
     for raw in items:
@@ -233,10 +291,10 @@ def process_json(file) -> Tuple[List[Dict[str, Any]], str, str, str]:
         first.append(it)
         scores.append(s)
 
-    # 2) Dinamik label fonksiyonu
+    # 2) Dynamic label function
     to_label = label_mapper_from_distribution(scores)
 
-    # 3) Label ata, low ise otomatik rewrite
+    # 3) Label, rewrite if 'low', then rescore
     processed = []
     for it in first:
         base_label = to_label(it["quality_before"]["score"])
@@ -244,7 +302,6 @@ def process_json(file) -> Tuple[List[Dict[str, Any]], str, str, str]:
 
         if base_label == "low":
             it = improve_smart(it)
-            # 4) yeniden skorla
             s2 = score_pair(it.get("question",""), it.get("answer",""))
             it["quality_after"] = {
                 "score": round(s2, 3),
@@ -252,7 +309,7 @@ def process_json(file) -> Tuple[List[Dict[str, Any]], str, str, str]:
             }
         processed.append(it)
 
-    # Summary tablo
+    # Summary table
     summary = []
     for idx, it in enumerate(processed):
         qb = it.get("quality_before", {})
@@ -266,12 +323,12 @@ def process_json(file) -> Tuple[List[Dict[str, Any]], str, str, str]:
             "question_preview": (it.get("question") or "")[:120]
         })
 
-    # İndirilebilir JSON
+    # Downloadable JSON
     tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json", mode="w", encoding="utf-8")
     json.dump(processed, tmp, indent=2, ensure_ascii=False)
     tmp.flush(); tmp.close()
 
-    # Önizleme
+    # Preview
     preview = json.dumps(processed[:50], indent=2, ensure_ascii=False)
     if len(processed) > 50:
         preview += "\n\n// NOTE: Showing first 50 items. Download full file below."
@@ -288,7 +345,7 @@ def process_json(file) -> Tuple[List[Dict[str, Any]], str, str, str]:
 with gr.Blocks(title="Q&A Quality Upgrader", theme=gr.themes.Soft()) as demo:
     gr.Markdown("## Q&A Quality Upgrader\nUpload your JSON. Low-quality items will be auto-rewritten and rescored.")
 
-    # ---- Minimal sample accordion (shows + download) ----
+    # ---- Minimal sample accordion (show + download) ----
     with gr.Accordion("Minimal sample JSON (only id, question, answer)", open=False):
         gr.Markdown("Upload a JSON **array of objects** with the following schema:")
         gr.Code(value=SAMPLE_JSON_MIN, language="json", lines=18, label="Minimal JSON example")