23.13-dars: NLP baholash va muammolar

TAKROR TURLARI:
  aniq takror:      "Xonasi toza, rahmat!"  ==  "xonasi toza, rahmat"
                    (registr, tinish belgisi, apostrof farq qiladi, xolos)
  deyarli takror:   "Takror: Xonasi toza, rahmat!" ~ "Xonasi toza, rahmat!"

TOPISH:
  1) xesh: md5( normallashtir(matn) dan harf/raqam/apostrof/bo'shliqdan
     boshqa hamma narsa olib tashlangan )  -> bir xil xesh = aniq takror
  2) TF-IDF kosinus > 0.9 -> deyarli takror          (n x n, bo'laklab)
  3) union-find: juftlar -> klasterlar

NIMA QILISH:
  noto'g'ri: tasodifiy bo'lish, takrorlar ikkala tomonga tushadi
  to'g'ri:   klaster bo'yicha bo'lish (GroupShuffleSplit / GroupKFold)
             yoki takrorlarni bo'lishdan OLDIN olib tashlash

NOTO'G'RI:
  X = TfidfVectorizer().fit_transform(hamma_matn)      # lug'at, IDF - testdan ham
  X = SelectKBest(chi2, k).fit_transform(X, hamma_y)   # TEST YORLIQLARI ko'rildi!
  cross_val_score(LogisticRegression(), X, y)

TO'G'RI:
  quvur = make_pipeline(TfidfVectorizer(...), SelectKBest(chi2, k),
                        LogisticRegression())
  cross_val_score(quvur, matn, y)       # har foldda hammasi faqat o'quvda fit

IKKI TUR LEAKAGE:
  nazoratsiz (vektorizator):  lug'at + IDF      -> odatda kichik, ishorasi noaniq
  nazoratli (chi2, target):   yorliq ko'riladi  -> tizimli optimizm

MUAMMO:
  bir muallif - ko'p sharh; muallifning:
    - uslubi (taxallus "@...", salom/xayr iborasi, imlo odati)
    - sinf moyilligi (doim maqtaydi yoki doim tanqid qiladi)
  KFold: muallifning sharhlari ham o'quvda, ham testda
    -> model "kim yozgan" -> "qanday baho" ni o'rganadi

TO'G'RI:
  GroupKFold(5).split(matn, y, groups=muallif)
  savol: "YANGI muallifning sharhi" uchun baho qanday?

TAQQOSLASH:
  KFold va GroupKFold foldlari har xil -> juftlash mumkin emas
  SE = sqrt(var_a / 5 + var_b / 5)

DOMEN = matn manbai: mavzu, platforma, davr, uslub
  telefon: "batareyasi tez tugaydi", "kamerasi tiniq"
  restoran: "taomi bemaza", "ofitsiant xushmuomala"
  -> sentiment so'zlarining katta qismi MAVZUGA XOS

O'LCHASH:
  ichki test:  o'sha mavzular, BOSHQA mualliflar
  tashqi test: yangi mavzular
  OOV ulushi:  test tokenlarining o'quv lug'atida yo'q qismi

YECHIM (arzonidan qimmatiga):
  1) yangi domendan oz miqdorda yorliqli ma'lumot qo'shish
  2) umumiy (domenga bog'liq bo'lmagan) belgilarga tayanish
  3) oldindan o'rgatilgan katta modellar (Transformerlar qismida)

KO'PCHILIK BAZAVIYSI: doim "ijobiy"
  aniqlik  = ijobiy ulushi (~0.65)      <- aldaydi
  makro-F1 = (0 + 0 + F1_ijobiy) / 3     <- darhol jazolaydi

MAKRO-F1 = uch sinf F1 ining ODDIY o'rtachasi (sinflar teng vaznda)
  kam sinf (neytral) ham ko'pchilik (ijobiy) bilan teng hissa qo'shadi

class_weight="balanced":
  kam sinf xatosi qimmatroq -> kam sinflar recall i oshadi
  ko'pchilik recall i kamayadi -> makro-F1 OSHISHI KAFOLATLANMAYDI
  tanlov validatsiyada, makro-F1 bo'yicha

XATO TAHLILI - "model QAYERDA adashadi?"
  har test misoliga belgilar:
    uzunlik (qisqa / o'rta / uzun), inkor bor/yo'q, imlo xatosi bor/yo'q,
    apostrof standart/nostandart, mavzu, haqiqiy sinf
  har segment: n, xato ulushi, 2*SE = 2 * sqrt(e(1-e)/n)

O'QISH QOIDASI:
  ikki segment farqi uning 2*SE lariga nisbatan katta bo'lsa -> jiddiy
  kichik segment (n < 30) -> faqat kuzatish, xulosa emas

KEYINGI QADAM:
  eng xavfli segmentdan 20-30 ta misolni QO'LDA o'qish
  -> yorliq xatosimi? model xatosimi? ma'lumot yetishmovchiligimi?

xesh = md5(re.sub(r"[^a-z0-9' ]", "", normallashtir(matn)))   # aniq takror
S = X[i:i + 1000] @ X.T; juftlar = S > 0.9                     # deyarli takror
kl = klasterlar(n, juftlar)                                    # union-find
GroupShuffleSplit(test_size=0.2).split(matn, y, groups=kl)     # takror bir tomonda
quvur = make_pipeline(TfidfVectorizer(...), SelectKBest(chi2, k), LogisticRegression())
cross_val_score(quvur, matn, y)                                # hammasi fold ichida
GroupKFold(5).split(matn, y, groups=muallif)                   # yangi muallif
oov = np.mean([s not in vektorizator.vocabulary_ for s in test_tokenlar])
f1_score(y, p, average="macro")                                # nomutanosib sinflar
e = xato[m].mean(); se = np.sqrt(e * (1 - e) / m.sum())        # xato jadvali

takrorlar: xesh + kosinus -> klaster -> klaster bo'yicha bo'lish
vektorizator va belgi tanlash -> Pipeline ichida (chi2 - tizimli optimizm)
muallif / mijoz -> GroupKFold (yangi manbaga umumlashish)
domen: ichki va tashqi test, OOV ulushi; yangi domendan oz yorliq
nomutanosib: makro-F1 + sinf bo'yicha recall; sinf vazni - qayta taqsimlash
xato tahlili: segment, n, xato, 2*SE -> eng xavfli segmentni qo'lda o'qish
har tuzoq: noto'g'ri va to'g'ri baho yonma-yon, farq va SE

"""Takroriy va deyarli takroriy matnlar: topish va leakage ni o'lchash."""

import hashlib
import re
from collections import Counter

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import GroupShuffleSplit, train_test_split
from sklearn.pipeline import make_pipeline

SINFLAR = ["salbiy", "neytral", "ijobiy"]
MAVZULAR = {   # mavzu: [(jihat, ijobiy sifatlar, salbiy sifatlar), ...], faktlar
    "telefon": ([("batareyasi", ["kuchli", "chidamli"], ["tez tugaydi", "zaif"]),
                 ("ekrani", ["yorqin", "tiniq"], ["xira", "tez chizildi"]),
                 ("kamerasi", ["zo'r", "tiniq"], ["loyqa", "bo'sh"]),
                 ("korpusi", ["mustahkam", "chiroyli"], ["mo'rt", "qiziydi"])],
                ["xotirasi 128 gb", "qora rangini oldim", "g'ilofi bilan keldi"]),
    "kiyim": ([("matosi", ["yumshoq", "pishiq"], ["dag'al", "yupqa"]),
               ("tikuvi", ["puxta", "tekis"], ["so'kilgan", "qiyshiq"]),
               ("o'lchami", ["mos", "aniq"], ["kichik", "noto'g'ri"]),
               ("rangi", ["chiroyli", "yorqin"], ["o'chib ketdi", "xira"])],
              ["44 o'lcham oldim", "qishga oldim", "ko'k rangda"]),
    "restoran": ([("taomi", ["mazali", "issiq"], ["bemaza", "sovuq"]),
                  ("ofitsiant", ["xushmuomala", "chaqqon"], ["qo'pol", "sust"]),
                  ("muhiti", ["shinam", "tinch"], ["shovqinli", "dim"]),
                  ("narxlari", ["arzon", "maqbul"], ["qimmat", "baland"])],
                 ["kechqurun bordik", "to'rt kishi edik", "plov buyurdik"]),
    "mehmonxona": ([("xonasi", ["toza", "keng"], ["iflos", "tor"]),
                    ("nonushtasi", ["to'yimli", "mazali"], ["bemaza", "kambag'al"]),
                    ("xodimlari", ["e'tiborli", "xushmuomala"], ["beparvo", "qo'pol"]),
                    ("joylashuvi", ["qulay", "markazda"], ["noqulay", "chekkada"])],
                   ["uch kecha qoldik", "ikkinchi qavatda edik", "oilaviy bordik"]),
}
UMUMIY = {2: ["yaxshi", "tavsiya qilaman", "yana kelaman", "mamnun qoldim", "juda yoqdi"],
          0: ["yomon", "tavsiya qilmayman", "boshqa kelmayman", "afsuslandim",
              "pulga arzimaydi"],
          1: ["o'rtacha", "normal", "bir marta sinab ko'rdim", "kutganimdek"]}
SALOM = ["", "", "assalomu alaykum", "salom hammaga", "xullas", "qisqasi", "rostini aytsam"]
XAYR = ["", "", "rahmat", "shu", "fikrim shu", "hammaga omad", "baho 5"]
APOSTROFLAR = ["'", "\u02bb", "\u2019", "`"]
BOGIN = ["di", "la", "ro", "ma", "zu", "ka", "ni", "sa", "bo", "tu", "gul", "an"]


def tanla(rng, royxat):
    return royxat[int(rng.integers(len(royxat)))]


def gap(rng, mavzu, qutb):
    """Bitta bo'lak; qutb: 2 ijobiy, 0 salbiy, 1 neytral (fakt)."""
    jihatlar, fakt = MAVZULAR[mavzu]
    if qutb == 1:
        return tanla(rng, fakt) if rng.random() < 0.6 else tanla(rng, UMUMIY[1])
    if rng.random() < 0.3:
        return tanla(rng, UMUMIY[qutb])
    jihat, ij, sl = tanla(rng, jihatlar)
    if rng.random() < 0.2:                                   # inkor: "tiniq emas"
        return f"{jihat} {tanla(rng, sl if qutb == 2 else ij)} emas"
    return f"{jihat} {tanla(rng, ij if qutb == 2 else sl)}"


def mualliflar(n, rng):
    """Har muallifning uslubi: taxallus, salom/xayr, apostrof, imlo, sinf moyilligi."""
    royxat = []
    for i in range(n):
        ism = "".join(tanla(rng, BOGIN) for _ in range(3)) + str(int(rng.integers(10, 99)))
        royxat.append({
            "id": i, "taxallus": "@" + ism if rng.random() < 0.5 else "",
            "salom": tanla(rng, SALOM), "xayr": tanla(rng, XAYR),
            "apostrof": tanla(rng, APOSTROFLAR), "imlo": float(rng.uniform(0, 0.08)),
            "moyillik": rng.dirichlet([0.2, 0.08, 0.5]),      # salbiy, neytral, ijobiy
            "mavzu": tanla(rng, list(MAVZULAR)),
        })
    return royxat


def imlo_buz(rng, soz, p):
    if len(soz) >= 5 and rng.random() < p:
        j = int(rng.integers(2, len(soz) - 1))
        return soz[:j] + soz[j + 1:] if rng.random() < 0.5 else soz[:j] + soz[j] + soz[j:]
    return soz


def sharh_yoz(rng, m, mavzu, yorliq):
    qutblar = [yorliq] * int(rng.integers(1, 3))
    if yorliq != 1 and rng.random() < 0.5:                   # qarama-qarshi bo'lak
        qutblar.append(2 - yorliq)
    if rng.random() < 0.5:
        qutblar.append(1)
    rng.shuffle(qutblar)
    bolaklar = [gap(rng, mavzu, q) for q in qutblar]
    matn = ", ".join([m["salom"]] * bool(m["salom"]) + bolaklar
                     + [m["xayr"]] * bool(m["xayr"]))
    matn = " ".join(imlo_buz(rng, s, m["imlo"]) for s in matn.split())
    matn = matn.replace("'", m["apostrof"]) + tanla(rng, [".", "!", ""])
    if m["taxallus"]:
        matn += " " + m["taxallus"]
    return matn[0].upper() + matn[1:]


def ozgartir(rng, matn):
    """Deyarli takror: registr, tinish belgisi, apostrof yoki bitta so'z o'zgaradi."""
    amal = int(rng.integers(4))
    if amal == 0:
        return matn.lower()
    if amal == 1:
        return matn.rstrip(".!") + tanla(rng, ["!!", " )", "..."])
    if amal == 2:
        return matn.replace("'", "\u2019").replace("\u02bb", "'")
    return tanla(rng, ["Yana yozaman: ", "Takror: ", "Muhim! "]) + matn


def korpus(seed=0, n_muallif=400, p_takror=0.25, p_shovqin=0.08):
    """Qaytaradi: DataFrame-ga o'xshash lug'at: matn, y, muallif, mavzu, takror."""
    rng = np.random.default_rng(seed)
    ml = mualliflar(n_muallif, rng)
    matn, y, muallif, mavzu, takror = [], [], [], [], []
    for m in ml:
        oldingi = []
        for _ in range(1 + int(rng.geometric(1 / 14))):
            if oldingi and rng.random() < p_takror:          # o'z sharhini qayta joylash
                j = tanla(rng, oldingi)
                t = matn[j] if rng.random() < 0.4 else ozgartir(rng, matn[j])
                matn.append(t)
                y.append(y[j])
                mavzu.append(mavzu[j])
                takror.append(True)
            else:
                mz = m["mavzu"] if rng.random() < 0.8 else tanla(rng, list(MAVZULAR))
                yl = int(rng.choice(3, p=m["moyillik"]))
                t = sharh_yoz(rng, m, mz, yl)
                if rng.random() < p_shovqin:
                    yl = int(rng.integers(3))
                matn.append(t)
                y.append(yl)
                mavzu.append(mz)
                takror.append(False)
                oldingi.append(len(matn) - 1)
            muallif.append(m["id"])
    return (np.array(matn, dtype=object), np.array(y), np.array(muallif),
            np.array(mavzu), np.array(takror))


def normallashtir(s):
    s = s.lower()
    for v in ("\u02bb", "\u2019", "\u2018", "\u02bc", "`"):
        s = s.replace(v, "'")
    s = re.sub(r"[^a-z0-9@' ]+", " ", s)
    return " ".join(s.split())


def tfidf_lr(C=5.0, class_weight=None):
    """Bazaviy quvur: normallashtirish -> TF-IDF (1-2 gram) -> LogisticRegression."""
    return make_pipeline(
        TfidfVectorizer(preprocessor=normallashtir, token_pattern=r"[^ ]+",
                        ngram_range=(1, 2), sublinear_tf=True, min_df=2),
        LogisticRegression(C=C, max_iter=3000, class_weight=class_weight))


def makro_f1(y, p):
    return f1_score(y, p, average="macro")


def juft_farq(a, b):
    """Juftlashgan farq: o'rtacha va SE (a, b - bir xil misollar/foldlar)."""
    d = np.asarray(a, float) - np.asarray(b, float)
    return d.mean(), d.std(ddof=1) / np.sqrt(len(d))


def xesh(matn):
    """Normallashtirilgan matn xeshi: registr, apostrof, tinish belgisi farqi yo'qoladi."""
    toza = re.sub(r"[^a-z0-9' ]", "", normallashtir(matn))
    return hashlib.md5(toza.encode()).hexdigest()


def yaqin_juftlar(matnlar, chegara=0.9, bolak=1000):
    """TF-IDF kosinus > chegara bo'lgan (i, j) juftlari, i < j."""
    X = TfidfVectorizer(preprocessor=normallashtir,
                        token_pattern=r"[^ ]+").fit_transform(matnlar)
    juftlar = []
    for s in range(0, X.shape[0], bolak):
        S = (X[s:s + bolak] @ X.T).toarray()
        for i, j in zip(*np.nonzero(S > chegara)):
            if s + i < j:
                juftlar.append((s + i, j))
    return juftlar


def klasterlar(n, juftlar):
    """Union-find: bir-biriga (deyarli) teng matnlar bitta klasterga."""
    ota = list(range(n))

    def top(i):
        while ota[i] != i:
            ota[i] = ota[ota[i]]
            i = ota[i]
        return i

    for i, j in juftlar:
        ota[top(i)] = top(j)
    return np.array([top(i) for i in range(n)])


def main() -> None:
    matn, y, muallif, mavzu, takror = korpus(seed=0)
    print("=== 1. Korpus ===")
    print(f"  sharhlar {len(y)}, mualliflar {len(set(muallif.tolist()))}")
    print("  sinflar: " + ", ".join(f"{s} {np.mean(y == k):.1%}"
                                    for k, s in enumerate(SINFLAR)))
    print(f"  generator qayta joylagan sharhlar: {takror.sum()} ({takror.mean():.1%})")

    print("\n=== 2. Takrorlarni topish ===")
    xeshlar = np.array([xesh(m) for m in matn])
    guruh = {}
    for i, h in enumerate(xeshlar):
        guruh.setdefault(h, []).append(i)
    aniq = [(g[0], j) for g in guruh.values() for j in g[1:]]
    yaqin = [(i, j) for i, j in yaqin_juftlar(matn) if xeshlar[i] != xeshlar[j]]
    print(f"  aniq takror (xesh bir xil): {len(aniq)} juft")
    print(f"  deyarli takror (xesh boshqa, kosinus > 0.9): {len(yaqin)} juft")
    kl = klasterlar(len(y), aniq + yaqin)
    egizakli = np.array([c > 1 for c in Counter(kl.tolist()).values()])
    topildi = np.isin(kl, [k for k, c in Counter(kl.tolist()).items() if c > 1])
    print(f"  klasterlar {len(set(kl.tolist()))}, egizakli klasterlar {egizakli.sum()}")
    print(f"  qayta joylanganlarning topilgani: {topildi[takror].mean():.1%}")
    print(f"  mustaqil yozilgan, lekin egizagi bor: {topildi[~takror].sum()} ta "
          "(qisqa bir xil sharhlar)")
    i, j = yaqin[0]
    for k in (i, j):
        print(f"    [{k}] {matn[k].encode('ascii', 'backslashreplace').decode()}")

    print("\n=== 3. Tasodifiy bo'lish: testning qanchasi o'quvda bor? ===")
    tr, te = train_test_split(np.arange(len(y)), test_size=0.2, stratify=y,
                              random_state=0)
    egizak = np.isin(kl[te], kl[tr])
    p = tfidf_lr().fit(matn[tr], y[tr]).predict(matn[te])
    print(f"  test {len(te)}, o'quvda egizagi bor: {egizak.sum()} ({egizak.mean():.1%})")
    print(f"  {'test qismi':<24} {'n':>5} {'aniqlik':>8} {'makro-F1':>9}")
    for nom, m in [("hammasi", np.ones(len(te), bool)), ("egizagi bor", egizak),
                   ("egizagi yo'q", ~egizak)]:
        print(f"  {nom:<24} {m.sum():>5} {np.mean(p[m] == y[te][m]):>8.3f} "
              f"{makro_f1(y[te][m], p[m]):>9.3f}")

    print("\n=== 4. Noto'g'ri va to'g'ri baho yonma-yon (5 bo'lish) ===")
    notogri, togri = [], []
    for seed in range(5):
        tr, te = train_test_split(np.arange(len(y)), test_size=0.2, stratify=y,
                                  random_state=seed)
        notogri.append(makro_f1(y[te], tfidf_lr().fit(matn[tr], y[tr]).predict(matn[te])))
        gss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=seed)
        tr, te = next(gss.split(matn, y, groups=kl))      # klaster bo'lib qoladi
        togri.append(makro_f1(y[te], tfidf_lr().fit(matn[tr], y[tr]).predict(matn[te])))
    print(f"  noto'g'ri (tasodifiy bo'lish):     {np.mean(notogri):.3f} "
          f"+- {np.std(notogri, ddof=1):.3f}")
    print(f"  to'g'ri (takror klasteri bo'yicha): {np.mean(togri):.3f} "
          f"+- {np.std(togri, ddof=1):.3f}")
    o, se = juft_farq(notogri, togri)
    print(f"  ortiqcha optimizm: {o:+.3f} (2*SE {2 * se:.3f}) -> "
          + ("sezilarli" if abs(o) > 2 * se else "sezilmas"))


if __name__ == "__main__":
    main()

=== 1. Korpus ===
  sharhlar 6132, mualliflar 400
  sinflar: salbiy 25.0%, neytral 13.8%, ijobiy 61.2%
  generator qayta joylagan sharhlar: 1471 (24.0%)

=== 2. Takrorlarni topish ===
  aniq takror (xesh bir xil): 1341 juft
  deyarli takror (xesh boshqa, kosinus > 0.9): 596 juft
  klasterlar 4488, egizakli klasterlar 1125
  qayta joylanganlarning topilgani: 94.0%
  mustaqil yozilgan, lekin egizagi bor: 1386 ta (qisqa bir xil sharhlar)
    [21] Qisqasi, joylashuvi chekkada emas, normal, joylashuvi noqulay!
    [5330] Qisqasi, joylashuvi noqulay emas, joylashuvi chekkada, fikrim shu!

=== 3. Tasodifiy bo'lish: testning qanchasi o'quvda bor? ===
  test 1227, o'quvda egizagi bor: 489 (39.9%)
  test qismi                   n  aniqlik  makro-F1
  hammasi                   1227    0.875     0.854
  egizagi bor                489    0.953     0.943
  egizagi yo'q               738    0.824     0.788

=== 4. Noto'g'ri va to'g'ri baho yonma-yon (5 bo'lish) ===
  noto'g'ri (tasodifiy bo'lish):     0.847 +- 0.008
  to'g'ri (takror klasteri bo'yicha): 0.808 +- 0.022
  ortiqcha optimizm: +0.040 (2*SE 0.025) -> sezilarli

"""Vektorizator va belgi tanlash leakage i: butun ma'lumotda fit va Pipeline ichida fit."""

import re

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import make_pipeline

SINFLAR = ["salbiy", "neytral", "ijobiy"]
MAVZULAR = {   # mavzu: [(jihat, ijobiy sifatlar, salbiy sifatlar), ...], faktlar
    "telefon": ([("batareyasi", ["kuchli", "chidamli"], ["tez tugaydi", "zaif"]),
                 ("ekrani", ["yorqin", "tiniq"], ["xira", "tez chizildi"]),
                 ("kamerasi", ["zo'r", "tiniq"], ["loyqa", "bo'sh"]),
                 ("korpusi", ["mustahkam", "chiroyli"], ["mo'rt", "qiziydi"])],
                ["xotirasi 128 gb", "qora rangini oldim", "g'ilofi bilan keldi"]),
    "kiyim": ([("matosi", ["yumshoq", "pishiq"], ["dag'al", "yupqa"]),
               ("tikuvi", ["puxta", "tekis"], ["so'kilgan", "qiyshiq"]),
               ("o'lchami", ["mos", "aniq"], ["kichik", "noto'g'ri"]),
               ("rangi", ["chiroyli", "yorqin"], ["o'chib ketdi", "xira"])],
              ["44 o'lcham oldim", "qishga oldim", "ko'k rangda"]),
    "restoran": ([("taomi", ["mazali", "issiq"], ["bemaza", "sovuq"]),
                  ("ofitsiant", ["xushmuomala", "chaqqon"], ["qo'pol", "sust"]),
                  ("muhiti", ["shinam", "tinch"], ["shovqinli", "dim"]),
                  ("narxlari", ["arzon", "maqbul"], ["qimmat", "baland"])],
                 ["kechqurun bordik", "to'rt kishi edik", "plov buyurdik"]),
    "mehmonxona": ([("xonasi", ["toza", "keng"], ["iflos", "tor"]),
                    ("nonushtasi", ["to'yimli", "mazali"], ["bemaza", "kambag'al"]),
                    ("xodimlari", ["e'tiborli", "xushmuomala"], ["beparvo", "qo'pol"]),
                    ("joylashuvi", ["qulay", "markazda"], ["noqulay", "chekkada"])],
                   ["uch kecha qoldik", "ikkinchi qavatda edik", "oilaviy bordik"]),
}
UMUMIY = {2: ["yaxshi", "tavsiya qilaman", "yana kelaman", "mamnun qoldim", "juda yoqdi"],
          0: ["yomon", "tavsiya qilmayman", "boshqa kelmayman", "afsuslandim",
              "pulga arzimaydi"],
          1: ["o'rtacha", "normal", "bir marta sinab ko'rdim", "kutganimdek"]}
SALOM = ["", "", "assalomu alaykum", "salom hammaga", "xullas", "qisqasi", "rostini aytsam"]
XAYR = ["", "", "rahmat", "shu", "fikrim shu", "hammaga omad", "baho 5"]
APOSTROFLAR = ["'", "\u02bb", "\u2019", "`"]
BOGIN = ["di", "la", "ro", "ma", "zu", "ka", "ni", "sa", "bo", "tu", "gul", "an"]


def tanla(rng, royxat):
    return royxat[int(rng.integers(len(royxat)))]


def gap(rng, mavzu, qutb):
    """Bitta bo'lak; qutb: 2 ijobiy, 0 salbiy, 1 neytral (fakt)."""
    jihatlar, fakt = MAVZULAR[mavzu]
    if qutb == 1:
        return tanla(rng, fakt) if rng.random() < 0.6 else tanla(rng, UMUMIY[1])
    if rng.random() < 0.3:
        return tanla(rng, UMUMIY[qutb])
    jihat, ij, sl = tanla(rng, jihatlar)
    if rng.random() < 0.2:                                   # inkor: "tiniq emas"
        return f"{jihat} {tanla(rng, sl if qutb == 2 else ij)} emas"
    return f"{jihat} {tanla(rng, ij if qutb == 2 else sl)}"


def mualliflar(n, rng):
    """Har muallifning uslubi: taxallus, salom/xayr, apostrof, imlo, sinf moyilligi."""
    royxat = []
    for i in range(n):
        ism = "".join(tanla(rng, BOGIN) for _ in range(3)) + str(int(rng.integers(10, 99)))
        royxat.append({
            "id": i, "taxallus": "@" + ism if rng.random() < 0.5 else "",
            "salom": tanla(rng, SALOM), "xayr": tanla(rng, XAYR),
            "apostrof": tanla(rng, APOSTROFLAR), "imlo": float(rng.uniform(0, 0.08)),
            "moyillik": rng.dirichlet([0.2, 0.08, 0.5]),      # salbiy, neytral, ijobiy
            "mavzu": tanla(rng, list(MAVZULAR)),
        })
    return royxat


def imlo_buz(rng, soz, p):
    if len(soz) >= 5 and rng.random() < p:
        j = int(rng.integers(2, len(soz) - 1))
        return soz[:j] + soz[j + 1:] if rng.random() < 0.5 else soz[:j] + soz[j] + soz[j:]
    return soz


def sharh_yoz(rng, m, mavzu, yorliq):
    qutblar = [yorliq] * int(rng.integers(1, 3))
    if yorliq != 1 and rng.random() < 0.5:                   # qarama-qarshi bo'lak
        qutblar.append(2 - yorliq)
    if rng.random() < 0.5:
        qutblar.append(1)
    rng.shuffle(qutblar)
    bolaklar = [gap(rng, mavzu, q) for q in qutblar]
    matn = ", ".join([m["salom"]] * bool(m["salom"]) + bolaklar
                     + [m["xayr"]] * bool(m["xayr"]))
    matn = " ".join(imlo_buz(rng, s, m["imlo"]) for s in matn.split())
    matn = matn.replace("'", m["apostrof"]) + tanla(rng, [".", "!", ""])
    if m["taxallus"]:
        matn += " " + m["taxallus"]
    return matn[0].upper() + matn[1:]


def ozgartir(rng, matn):
    """Deyarli takror: registr, tinish belgisi, apostrof yoki bitta so'z o'zgaradi."""
    amal = int(rng.integers(4))
    if amal == 0:
        return matn.lower()
    if amal == 1:
        return matn.rstrip(".!") + tanla(rng, ["!!", " )", "..."])
    if amal == 2:
        return matn.replace("'", "\u2019").replace("\u02bb", "'")
    return tanla(rng, ["Yana yozaman: ", "Takror: ", "Muhim! "]) + matn


def korpus(seed=0, n_muallif=400, p_takror=0.25, p_shovqin=0.08):
    """Qaytaradi: DataFrame-ga o'xshash lug'at: matn, y, muallif, mavzu, takror."""
    rng = np.random.default_rng(seed)
    ml = mualliflar(n_muallif, rng)
    matn, y, muallif, mavzu, takror = [], [], [], [], []
    for m in ml:
        oldingi = []
        for _ in range(1 + int(rng.geometric(1 / 14))):
            if oldingi and rng.random() < p_takror:          # o'z sharhini qayta joylash
                j = tanla(rng, oldingi)
                t = matn[j] if rng.random() < 0.4 else ozgartir(rng, matn[j])
                matn.append(t)
                y.append(y[j])
                mavzu.append(mavzu[j])
                takror.append(True)
            else:
                mz = m["mavzu"] if rng.random() < 0.8 else tanla(rng, list(MAVZULAR))
                yl = int(rng.choice(3, p=m["moyillik"]))
                t = sharh_yoz(rng, m, mz, yl)
                if rng.random() < p_shovqin:
                    yl = int(rng.integers(3))
                matn.append(t)
                y.append(yl)
                mavzu.append(mz)
                takror.append(False)
                oldingi.append(len(matn) - 1)
            muallif.append(m["id"])
    return (np.array(matn, dtype=object), np.array(y), np.array(muallif),
            np.array(mavzu), np.array(takror))


def normallashtir(s):
    s = s.lower()
    for v in ("\u02bb", "\u2019", "\u2018", "\u02bc", "`"):
        s = s.replace(v, "'")
    s = re.sub(r"[^a-z0-9@' ]+", " ", s)
    return " ".join(s.split())


def tfidf_lr(C=5.0, class_weight=None):
    """Bazaviy quvur: normallashtirish -> TF-IDF (1-2 gram) -> LogisticRegression."""
    return make_pipeline(
        TfidfVectorizer(preprocessor=normallashtir, token_pattern=r"[^ ]+",
                        ngram_range=(1, 2), sublinear_tf=True, min_df=2),
        LogisticRegression(C=C, max_iter=3000, class_weight=class_weight))


def makro_f1(y, p):
    return f1_score(y, p, average="macro")


def juft_farq(a, b):
    """Juftlashgan farq: o'rtacha va SE (a, b - bir xil misollar/foldlar)."""
    d = np.asarray(a, float) - np.asarray(b, float)
    return d.mean(), d.std(ddof=1) / np.sqrt(len(d))


def vektorizator(ngram=(1, 2)):
    return TfidfVectorizer(preprocessor=normallashtir, token_pattern=r"[^ ]+",
                           ngram_range=ngram, sublinear_tf=True, min_df=2)


def cv_baho(matn, y, notogri, tanlash=None, n_splits=5):
    """notogri=True: vektorizator (va tanlash) BUTUN ma'lumotda fit qilinadi."""
    cv = StratifiedKFold(n_splits, shuffle=True, random_state=0)
    ballar = []
    if notogri:
        X = vektorizator((1, 3)).fit_transform(matn)
        if tanlash:
            X = SelectKBest(chi2, k=tanlash).fit_transform(X, y)   # y ham ko'rildi!
    for tr, te in cv.split(matn, y):
        if notogri:
            model = LogisticRegression(C=5.0, max_iter=3000).fit(X[tr], y[tr])
            p = model.predict(X[te])
        else:
            qadamlar = [vektorizator((1, 3))]
            if tanlash:
                qadamlar.append(SelectKBest(chi2, k=tanlash))
            qadamlar.append(LogisticRegression(C=5.0, max_iter=3000))
            p = make_pipeline(*qadamlar).fit(matn[tr], y[tr]).predict(matn[te])
        ballar.append(makro_f1(y[te], p))
    return np.array(ballar)


def main() -> None:
    matn, y, muallif, mavzu, takror = korpus(seed=0)
    birinchi = np.flatnonzero(~takror)                 # takrorlar olib tashlandi
    matn, y = matn[birinchi], y[birinchi]
    print("=== 1. Vektorizatorni qayerda fit qilamiz? ===")
    tr, te = np.arange(0, 4000), np.arange(4000, len(y))
    v_tr = vektorizator().fit(matn[tr])
    v_all = vektorizator().fit(matn)
    yangi = sorted(set(v_all.vocabulary_) - set(v_tr.vocabulary_))
    print(f"  o'quvda fit: lug'at {len(v_tr.vocabulary_)}")
    print(f"  hammasida fit: lug'at {len(v_all.vocabulary_)} "
          f"(+{len(yangi)} faqat testdan kelgan belgi)")
    umumiy = sorted(set(v_tr.vocabulary_) & set(v_all.vocabulary_))
    i1 = v_tr.idf_[[v_tr.vocabulary_[s] for s in umumiy]]
    i2 = v_all.idf_[[v_all.vocabulary_[s] for s in umumiy]]
    print(f"  umumiy belgilarda IDF farqi: o'rtacha {np.abs(i1 - i2).mean():.4f}, "
          f"eng katta {np.abs(i1 - i2).max():.4f}")

    print("\n=== 2. Faqat vektorizator: noto'g'ri va to'g'ri (5 fold) ===")
    a = cv_baho(matn, y, notogri=True)
    b = cv_baho(matn, y, notogri=False)
    o, se = juft_farq(a, b)
    print(f"  noto'g'ri (hammasida fit):  {a.mean():.4f}")
    print(f"  to'g'ri (Pipeline ichida):  {b.mean():.4f}")
    print(f"  farq {o:+.4f}, 2*SE {2 * se:.4f} -> "
          + ("sezilarli" if abs(o) > 2 * se else "sezilmas"))

    print("\n=== 3. Belgi tanlash (chi2) - kichik ma'lumot, 600 sharh ===")
    rng = np.random.default_rng(1)
    kichik = rng.choice(len(y), 600, replace=False)
    m6, y6 = matn[kichik], y[kichik]
    print("  " + "yorliqlar".ljust(16) + "noto'g'ri".rjust(9) + "to'g'ri".rjust(9)
          + "farq".rjust(8) + "2*SE".rjust(7))
    for nom, yy in [("haqiqiy", y6), ("aralashtirilgan", rng.permutation(y6))]:
        a = cv_baho(m6, yy, notogri=True, tanlash=300)
        b = cv_baho(m6, yy, notogri=False, tanlash=300)
        o, se = juft_farq(a, b)
        print(f"  {nom:<16}{a.mean():>9.3f} {b.mean():>8.3f} {o:>+7.3f} {2 * se:>6.3f}")
    print("  aralashtirilgan yorliqda to'g'ri baho ~ tasodif (3 sinf, makro-F1)")


if __name__ == "__main__":
    main()

=== 1. Vektorizatorni qayerda fit qilamiz? ===
  o'quvda fit: lug'at 2769
  hammasida fit: lug'at 3054 (+285 faqat testdan kelgan belgi)
  umumiy belgilarda IDF farqi: o'rtacha 0.1269, eng katta 0.9457

=== 2. Faqat vektorizator: noto'g'ri va to'g'ri (5 fold) ===
  noto'g'ri (hammasida fit):  0.7837
  to'g'ri (Pipeline ichida):  0.7894
  farq -0.0057, 2*SE 0.0075 -> sezilmas

=== 3. Belgi tanlash (chi2) - kichik ma'lumot, 600 sharh ===
  yorliqlar       noto'g'ri  to'g'ri    farq   2*SE
  haqiqiy             0.693    0.650  +0.043  0.052
  aralashtirilgan     0.332    0.294  +0.037  0.029
  aralashtirilgan yorliqda to'g'ri baho ~ tasodif (3 sinf, makro-F1)

"""Muallif bo'yicha GroupKFold va domen siljishi."""

import re

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import GroupKFold, GroupShuffleSplit, KFold
from sklearn.pipeline import make_pipeline

SINFLAR = ["salbiy", "neytral", "ijobiy"]
MAVZULAR = {   # mavzu: [(jihat, ijobiy sifatlar, salbiy sifatlar), ...], faktlar
    "telefon": ([("batareyasi", ["kuchli", "chidamli"], ["tez tugaydi", "zaif"]),
                 ("ekrani", ["yorqin", "tiniq"], ["xira", "tez chizildi"]),
                 ("kamerasi", ["zo'r", "tiniq"], ["loyqa", "bo'sh"]),
                 ("korpusi", ["mustahkam", "chiroyli"], ["mo'rt", "qiziydi"])],
                ["xotirasi 128 gb", "qora rangini oldim", "g'ilofi bilan keldi"]),
    "kiyim": ([("matosi", ["yumshoq", "pishiq"], ["dag'al", "yupqa"]),
               ("tikuvi", ["puxta", "tekis"], ["so'kilgan", "qiyshiq"]),
               ("o'lchami", ["mos", "aniq"], ["kichik", "noto'g'ri"]),
               ("rangi", ["chiroyli", "yorqin"], ["o'chib ketdi", "xira"])],
              ["44 o'lcham oldim", "qishga oldim", "ko'k rangda"]),
    "restoran": ([("taomi", ["mazali", "issiq"], ["bemaza", "sovuq"]),
                  ("ofitsiant", ["xushmuomala", "chaqqon"], ["qo'pol", "sust"]),
                  ("muhiti", ["shinam", "tinch"], ["shovqinli", "dim"]),
                  ("narxlari", ["arzon", "maqbul"], ["qimmat", "baland"])],
                 ["kechqurun bordik", "to'rt kishi edik", "plov buyurdik"]),
    "mehmonxona": ([("xonasi", ["toza", "keng"], ["iflos", "tor"]),
                    ("nonushtasi", ["to'yimli", "mazali"], ["bemaza", "kambag'al"]),
                    ("xodimlari", ["e'tiborli", "xushmuomala"], ["beparvo", "qo'pol"]),
                    ("joylashuvi", ["qulay", "markazda"], ["noqulay", "chekkada"])],
                   ["uch kecha qoldik", "ikkinchi qavatda edik", "oilaviy bordik"]),
}
UMUMIY = {2: ["yaxshi", "tavsiya qilaman", "yana kelaman", "mamnun qoldim", "juda yoqdi"],
          0: ["yomon", "tavsiya qilmayman", "boshqa kelmayman", "afsuslandim",
              "pulga arzimaydi"],
          1: ["o'rtacha", "normal", "bir marta sinab ko'rdim", "kutganimdek"]}
SALOM = ["", "", "assalomu alaykum", "salom hammaga", "xullas", "qisqasi", "rostini aytsam"]
XAYR = ["", "", "rahmat", "shu", "fikrim shu", "hammaga omad", "baho 5"]
APOSTROFLAR = ["'", "\u02bb", "\u2019", "`"]
BOGIN = ["di", "la", "ro", "ma", "zu", "ka", "ni", "sa", "bo", "tu", "gul", "an"]


def tanla(rng, royxat):
    return royxat[int(rng.integers(len(royxat)))]


def gap(rng, mavzu, qutb):
    """Bitta bo'lak; qutb: 2 ijobiy, 0 salbiy, 1 neytral (fakt)."""
    jihatlar, fakt = MAVZULAR[mavzu]
    if qutb == 1:
        return tanla(rng, fakt) if rng.random() < 0.6 else tanla(rng, UMUMIY[1])
    if rng.random() < 0.3:
        return tanla(rng, UMUMIY[qutb])
    jihat, ij, sl = tanla(rng, jihatlar)
    if rng.random() < 0.2:                                   # inkor: "tiniq emas"
        return f"{jihat} {tanla(rng, sl if qutb == 2 else ij)} emas"
    return f"{jihat} {tanla(rng, ij if qutb == 2 else sl)}"


def mualliflar(n, rng):
    """Har muallifning uslubi: taxallus, salom/xayr, apostrof, imlo, sinf moyilligi."""
    royxat = []
    for i in range(n):
        ism = "".join(tanla(rng, BOGIN) for _ in range(3)) + str(int(rng.integers(10, 99)))
        royxat.append({
            "id": i, "taxallus": "@" + ism if rng.random() < 0.5 else "",
            "salom": tanla(rng, SALOM), "xayr": tanla(rng, XAYR),
            "apostrof": tanla(rng, APOSTROFLAR), "imlo": float(rng.uniform(0, 0.08)),
            "moyillik": rng.dirichlet([0.2, 0.08, 0.5]),      # salbiy, neytral, ijobiy
            "mavzu": tanla(rng, list(MAVZULAR)),
        })
    return royxat


def imlo_buz(rng, soz, p):
    if len(soz) >= 5 and rng.random() < p:
        j = int(rng.integers(2, len(soz) - 1))
        return soz[:j] + soz[j + 1:] if rng.random() < 0.5 else soz[:j] + soz[j] + soz[j:]
    return soz


def sharh_yoz(rng, m, mavzu, yorliq):
    qutblar = [yorliq] * int(rng.integers(1, 3))
    if yorliq != 1 and rng.random() < 0.5:                   # qarama-qarshi bo'lak
        qutblar.append(2 - yorliq)
    if rng.random() < 0.5:
        qutblar.append(1)
    rng.shuffle(qutblar)
    bolaklar = [gap(rng, mavzu, q) for q in qutblar]
    matn = ", ".join([m["salom"]] * bool(m["salom"]) + bolaklar
                     + [m["xayr"]] * bool(m["xayr"]))
    matn = " ".join(imlo_buz(rng, s, m["imlo"]) for s in matn.split())
    matn = matn.replace("'", m["apostrof"]) + tanla(rng, [".", "!", ""])
    if m["taxallus"]:
        matn += " " + m["taxallus"]
    return matn[0].upper() + matn[1:]


def ozgartir(rng, matn):
    """Deyarli takror: registr, tinish belgisi, apostrof yoki bitta so'z o'zgaradi."""
    amal = int(rng.integers(4))
    if amal == 0:
        return matn.lower()
    if amal == 1:
        return matn.rstrip(".!") + tanla(rng, ["!!", " )", "..."])
    if amal == 2:
        return matn.replace("'", "\u2019").replace("\u02bb", "'")
    return tanla(rng, ["Yana yozaman: ", "Takror: ", "Muhim! "]) + matn


def korpus(seed=0, n_muallif=400, p_takror=0.25, p_shovqin=0.08):
    """Qaytaradi: DataFrame-ga o'xshash lug'at: matn, y, muallif, mavzu, takror."""
    rng = np.random.default_rng(seed)
    ml = mualliflar(n_muallif, rng)
    matn, y, muallif, mavzu, takror = [], [], [], [], []
    for m in ml:
        oldingi = []
        for _ in range(1 + int(rng.geometric(1 / 14))):
            if oldingi and rng.random() < p_takror:          # o'z sharhini qayta joylash
                j = tanla(rng, oldingi)
                t = matn[j] if rng.random() < 0.4 else ozgartir(rng, matn[j])
                matn.append(t)
                y.append(y[j])
                mavzu.append(mavzu[j])
                takror.append(True)
            else:
                mz = m["mavzu"] if rng.random() < 0.8 else tanla(rng, list(MAVZULAR))
                yl = int(rng.choice(3, p=m["moyillik"]))
                t = sharh_yoz(rng, m, mz, yl)
                if rng.random() < p_shovqin:
                    yl = int(rng.integers(3))
                matn.append(t)
                y.append(yl)
                mavzu.append(mz)
                takror.append(False)
                oldingi.append(len(matn) - 1)
            muallif.append(m["id"])
    return (np.array(matn, dtype=object), np.array(y), np.array(muallif),
            np.array(mavzu), np.array(takror))


def normallashtir(s):
    s = s.lower()
    for v in ("\u02bb", "\u2019", "\u2018", "\u02bc", "`"):
        s = s.replace(v, "'")
    s = re.sub(r"[^a-z0-9@' ]+", " ", s)
    return " ".join(s.split())


def tfidf_lr(C=5.0, class_weight=None):
    """Bazaviy quvur: normallashtirish -> TF-IDF (1-2 gram) -> LogisticRegression."""
    return make_pipeline(
        TfidfVectorizer(preprocessor=normallashtir, token_pattern=r"[^ ]+",
                        ngram_range=(1, 2), sublinear_tf=True, min_df=2),
        LogisticRegression(C=C, max_iter=3000, class_weight=class_weight))


def makro_f1(y, p):
    return f1_score(y, p, average="macro")


def juft_farq(a, b):
    """Juftlashgan farq: o'rtacha va SE (a, b - bir xil misollar/foldlar)."""
    d = np.asarray(a, float) - np.asarray(b, float)
    return d.mean(), d.std(ddof=1) / np.sqrt(len(d))


def cv_f1(matn, y, cv, guruh=None):
    ballar = []
    for tr, te in cv.split(matn, y, guruh):
        ballar.append(makro_f1(y[te], tfidf_lr().fit(matn[tr], y[tr]).predict(matn[te])))
    return np.array(ballar)


def main() -> None:
    matn, y, muallif, mavzu, takror = korpus(seed=0)
    ok = ~takror                                         # qayta joylanganlar olib tashlandi
    matn, y, muallif, mavzu = matn[ok], y[ok], muallif[ok], mavzu[ok]

    print("=== 1. Mualliflar ===")
    soni = np.bincount(muallif)
    soni = soni[soni > 0]
    print(f"  sharhlar {len(y)}, mualliflar {len(soni)}; bitta muallifda: "
          f"median {int(np.median(soni))}, eng ko'p {soni.max()}")
    top = np.sort(soni)[::-1]
    print(f"  eng faol 20 muallif sharhlarning {top[:20].sum() / len(y):.1%} ini yozgan")
    ustun = [np.bincount(y[muallif == a], minlength=3).max() / np.sum(muallif == a)
             for a in np.unique(muallif) if np.sum(muallif == a) >= 5]
    print(f"  5+ sharhli mualliflarning {np.mean(np.array(ustun) >= 0.8):.1%} ida "
          "sharhlarning 80%+ i bitta sinfda")
    print(f"  taxallus bilan imzolangan sharhlar: {np.mean(['@' in m for m in matn]):.1%}")

    print("\n=== 2. KFold va GroupKFold (muallif bo'yicha), 5 fold ===")
    a = cv_f1(matn, y, KFold(5, shuffle=True, random_state=0))
    b = cv_f1(matn, y, GroupKFold(5), muallif)
    print(f"  noto'g'ri KFold:        {a.round(3).tolist()}  o'rtacha {a.mean():.3f}")
    print(f"  to'g'ri GroupKFold:     {b.round(3).tolist()}  o'rtacha {b.mean():.3f}")
    se = np.sqrt(a.var(ddof=1) / 5 + b.var(ddof=1) / 5)   # foldlar har xil - juft emas
    farq = a.mean() - b.mean()
    print(f"  farq {farq:+.3f}, 2*SE {2 * se:.3f} -> "
          + ("sezilarli" if abs(farq) > 2 * se else "sezilmas"))

    print("\n=== 3. KFold modeli nimani o'rgandi? ===")
    model = tfidf_lr().fit(matn, y)
    v, lr = model.steps[0][1], model.steps[1][1]
    nomlar = v.get_feature_names_out()
    for k in (2, 0):
        eng = np.argsort(lr.coef_[k])[::-1][:30]
        taxallus = [nomlar[j] for j in eng if "@" in nomlar[j]]
        print(f"  {SINFLAR[k]} uchun eng kuchli 30 belgidan {len(taxallus)} tasi taxallus, "
              f"masalan {taxallus[:2]}")

    print("\n=== 4. Domen siljishi: telefon+kiyim da o'rgatib ... ===")
    manba = np.isin(mavzu, ["telefon", "kiyim"])
    idx = np.flatnonzero(manba)
    gss = GroupShuffleSplit(n_splits=1, test_size=0.25, random_state=0)
    tr_i, te_i = next(gss.split(idx, groups=muallif[idx]))
    tr, te_ichki = idx[tr_i], idx[te_i]
    tashqi = np.flatnonzero(~manba & ~np.isin(muallif, muallif[tr]))
    model = tfidf_lr().fit(matn[tr], y[tr])
    lugat = model.steps[0][1].vocabulary_
    print(f"  {'test':<28} {'n':>5} {'OOV':>6} {'aniqlik':>8} {'makro-F1':>9}")
    for nom, te in [("ichki (telefon+kiyim)", te_ichki),
                    ("tashqi (restoran+mehmonxona)", tashqi)]:
        tok = [s for m in matn[te] for s in normallashtir(m).split()]
        oov = np.mean([s not in lugat for s in tok])
        p = model.predict(matn[te])
        print(f"  {nom:<28} {len(te):>5} {oov:>6.1%} {np.mean(p == y[te]):>8.3f} "
              f"{makro_f1(y[te], p):>9.3f}")

    print("\n=== 5. Yechim: yangi domendan 300 ta yorliqli sharh qo'shish ===")
    tashqi_muallif = np.unique(muallif[tashqi])
    rng = np.random.default_rng(0)
    yangi_m = rng.permutation(tashqi_muallif)[: len(tashqi_muallif) // 4]
    qosh = np.flatnonzero(np.isin(muallif, yangi_m) & ~manba)[:300]
    sinov = tashqi[~np.isin(muallif[tashqi], yangi_m)]
    p1 = tfidf_lr().fit(matn[tr], y[tr]).predict(matn[sinov])
    tr2 = np.concatenate([tr, qosh])
    p2 = tfidf_lr().fit(matn[tr2], y[tr2]).predict(matn[sinov])
    print(f"  sinov: {len(sinov)} ta tashqi sharh (qo'shilganlarning mualliflari yo'q)")
    print(f"  faqat manba domen:      aniqlik {np.mean(p1 == y[sinov]):.3f}, "
          f"makro-F1 {makro_f1(y[sinov], p1):.3f}")
    print(f"  + 300 yangi domen:      aniqlik {np.mean(p2 == y[sinov]):.3f}, "
          f"makro-F1 {makro_f1(y[sinov], p2):.3f}")
    o, se = juft_farq(p2 == y[sinov], p1 == y[sinov])
    print(f"  juftlashgan farq (aniqlik) {o:+.3f}, 2*SE {2 * se:.3f} -> "
          + ("sezilarli" if abs(o) > 2 * se else "sezilmas"))


if __name__ == "__main__":
    main()

=== 1. Mualliflar ===
  sharhlar 4661, mualliflar 400; bitta muallifda: median 8, eng ko'p 61
  eng faol 20 muallif sharhlarning 17.9% ini yozgan
  5+ sharhli mualliflarning 63.7% ida sharhlarning 80%+ i bitta sinfda
  taxallus bilan imzolangan sharhlar: 48.8%

=== 2. KFold va GroupKFold (muallif bo'yicha), 5 fold ===
  noto'g'ri KFold:        [0.79, 0.789, 0.803, 0.809, 0.805]  o'rtacha 0.799
  to'g'ri GroupKFold:     [0.724, 0.718, 0.768, 0.791, 0.757]  o'rtacha 0.752
  farq +0.048, 2*SE 0.028 -> sezilarli

=== 3. KFold modeli nimani o'rgandi? ===
  ijobiy uchun eng kuchli 30 belgidan 1 tasi taxallus, masalan ['@tuladi79']
  salbiy uchun eng kuchli 30 belgidan 4 tasi taxallus, masalan ['5 @dianma81', '@dianma81']

=== 4. Domen siljishi: telefon+kiyim da o'rgatib ... ===
  test                             n    OOV  aniqlik  makro-F1
  ichki (telefon+kiyim)          578   8.6%    0.761     0.719
  tashqi (restoran+mehmonxona)   825  52.0%    0.588     0.532

=== 5. Yechim: yangi domendan 300 ta yorliqli sharh qo'shish ===
  sinov: 651 ta tashqi sharh (qo'shilganlarning mualliflari yo'q)
  faqat manba domen:      aniqlik 0.591, makro-F1 0.539
  + 300 yangi domen:      aniqlik 0.676, makro-F1 0.651
  juftlashgan farq (aniqlik) +0.084, 2*SE 0.041 -> sezilarli

"""Nomutanosib sinflar, makro-F1 va xato tahlili jadvali."""

import re

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import GroupShuffleSplit
from sklearn.pipeline import make_pipeline

SINFLAR = ["salbiy", "neytral", "ijobiy"]
MAVZULAR = {   # mavzu: [(jihat, ijobiy sifatlar, salbiy sifatlar), ...], faktlar
    "telefon": ([("batareyasi", ["kuchli", "chidamli"], ["tez tugaydi", "zaif"]),
                 ("ekrani", ["yorqin", "tiniq"], ["xira", "tez chizildi"]),
                 ("kamerasi", ["zo'r", "tiniq"], ["loyqa", "bo'sh"]),
                 ("korpusi", ["mustahkam", "chiroyli"], ["mo'rt", "qiziydi"])],
                ["xotirasi 128 gb", "qora rangini oldim", "g'ilofi bilan keldi"]),
    "kiyim": ([("matosi", ["yumshoq", "pishiq"], ["dag'al", "yupqa"]),
               ("tikuvi", ["puxta", "tekis"], ["so'kilgan", "qiyshiq"]),
               ("o'lchami", ["mos", "aniq"], ["kichik", "noto'g'ri"]),
               ("rangi", ["chiroyli", "yorqin"], ["o'chib ketdi", "xira"])],
              ["44 o'lcham oldim", "qishga oldim", "ko'k rangda"]),
    "restoran": ([("taomi", ["mazali", "issiq"], ["bemaza", "sovuq"]),
                  ("ofitsiant", ["xushmuomala", "chaqqon"], ["qo'pol", "sust"]),
                  ("muhiti", ["shinam", "tinch"], ["shovqinli", "dim"]),
                  ("narxlari", ["arzon", "maqbul"], ["qimmat", "baland"])],
                 ["kechqurun bordik", "to'rt kishi edik", "plov buyurdik"]),
    "mehmonxona": ([("xonasi", ["toza", "keng"], ["iflos", "tor"]),
                    ("nonushtasi", ["to'yimli", "mazali"], ["bemaza", "kambag'al"]),
                    ("xodimlari", ["e'tiborli", "xushmuomala"], ["beparvo", "qo'pol"]),
                    ("joylashuvi", ["qulay", "markazda"], ["noqulay", "chekkada"])],
                   ["uch kecha qoldik", "ikkinchi qavatda edik", "oilaviy bordik"]),
}
UMUMIY = {2: ["yaxshi", "tavsiya qilaman", "yana kelaman", "mamnun qoldim", "juda yoqdi"],
          0: ["yomon", "tavsiya qilmayman", "boshqa kelmayman", "afsuslandim",
              "pulga arzimaydi"],
          1: ["o'rtacha", "normal", "bir marta sinab ko'rdim", "kutganimdek"]}
SALOM = ["", "", "assalomu alaykum", "salom hammaga", "xullas", "qisqasi", "rostini aytsam"]
XAYR = ["", "", "rahmat", "shu", "fikrim shu", "hammaga omad", "baho 5"]
APOSTROFLAR = ["'", "\u02bb", "\u2019", "`"]
BOGIN = ["di", "la", "ro", "ma", "zu", "ka", "ni", "sa", "bo", "tu", "gul", "an"]


def tanla(rng, royxat):
    return royxat[int(rng.integers(len(royxat)))]


def gap(rng, mavzu, qutb):
    """Bitta bo'lak; qutb: 2 ijobiy, 0 salbiy, 1 neytral (fakt)."""
    jihatlar, fakt = MAVZULAR[mavzu]
    if qutb == 1:
        return tanla(rng, fakt) if rng.random() < 0.6 else tanla(rng, UMUMIY[1])
    if rng.random() < 0.3:
        return tanla(rng, UMUMIY[qutb])
    jihat, ij, sl = tanla(rng, jihatlar)
    if rng.random() < 0.2:                                   # inkor: "tiniq emas"
        return f"{jihat} {tanla(rng, sl if qutb == 2 else ij)} emas"
    return f"{jihat} {tanla(rng, ij if qutb == 2 else sl)}"


def mualliflar(n, rng):
    """Har muallifning uslubi: taxallus, salom/xayr, apostrof, imlo, sinf moyilligi."""
    royxat = []
    for i in range(n):
        ism = "".join(tanla(rng, BOGIN) for _ in range(3)) + str(int(rng.integers(10, 99)))
        royxat.append({
            "id": i, "taxallus": "@" + ism if rng.random() < 0.5 else "",
            "salom": tanla(rng, SALOM), "xayr": tanla(rng, XAYR),
            "apostrof": tanla(rng, APOSTROFLAR), "imlo": float(rng.uniform(0, 0.08)),
            "moyillik": rng.dirichlet([0.2, 0.08, 0.5]),      # salbiy, neytral, ijobiy
            "mavzu": tanla(rng, list(MAVZULAR)),
        })
    return royxat


def imlo_buz(rng, soz, p):
    if len(soz) >= 5 and rng.random() < p:
        j = int(rng.integers(2, len(soz) - 1))
        return soz[:j] + soz[j + 1:] if rng.random() < 0.5 else soz[:j] + soz[j] + soz[j:]
    return soz


def sharh_yoz(rng, m, mavzu, yorliq):
    qutblar = [yorliq] * int(rng.integers(1, 3))
    if yorliq != 1 and rng.random() < 0.5:                   # qarama-qarshi bo'lak
        qutblar.append(2 - yorliq)
    if rng.random() < 0.5:
        qutblar.append(1)
    rng.shuffle(qutblar)
    bolaklar = [gap(rng, mavzu, q) for q in qutblar]
    matn = ", ".join([m["salom"]] * bool(m["salom"]) + bolaklar
                     + [m["xayr"]] * bool(m["xayr"]))
    matn = " ".join(imlo_buz(rng, s, m["imlo"]) for s in matn.split())
    matn = matn.replace("'", m["apostrof"]) + tanla(rng, [".", "!", ""])
    if m["taxallus"]:
        matn += " " + m["taxallus"]
    return matn[0].upper() + matn[1:]


def ozgartir(rng, matn):
    """Deyarli takror: registr, tinish belgisi, apostrof yoki bitta so'z o'zgaradi."""
    amal = int(rng.integers(4))
    if amal == 0:
        return matn.lower()
    if amal == 1:
        return matn.rstrip(".!") + tanla(rng, ["!!", " )", "..."])
    if amal == 2:
        return matn.replace("'", "\u2019").replace("\u02bb", "'")
    return tanla(rng, ["Yana yozaman: ", "Takror: ", "Muhim! "]) + matn


def korpus(seed=0, n_muallif=400, p_takror=0.25, p_shovqin=0.08):
    """Qaytaradi: DataFrame-ga o'xshash lug'at: matn, y, muallif, mavzu, takror."""
    rng = np.random.default_rng(seed)
    ml = mualliflar(n_muallif, rng)
    matn, y, muallif, mavzu, takror = [], [], [], [], []
    for m in ml:
        oldingi = []
        for _ in range(1 + int(rng.geometric(1 / 14))):
            if oldingi and rng.random() < p_takror:          # o'z sharhini qayta joylash
                j = tanla(rng, oldingi)
                t = matn[j] if rng.random() < 0.4 else ozgartir(rng, matn[j])
                matn.append(t)
                y.append(y[j])
                mavzu.append(mavzu[j])
                takror.append(True)
            else:
                mz = m["mavzu"] if rng.random() < 0.8 else tanla(rng, list(MAVZULAR))
                yl = int(rng.choice(3, p=m["moyillik"]))
                t = sharh_yoz(rng, m, mz, yl)
                if rng.random() < p_shovqin:
                    yl = int(rng.integers(3))
                matn.append(t)
                y.append(yl)
                mavzu.append(mz)
                takror.append(False)
                oldingi.append(len(matn) - 1)
            muallif.append(m["id"])
    return (np.array(matn, dtype=object), np.array(y), np.array(muallif),
            np.array(mavzu), np.array(takror))


def normallashtir(s):
    s = s.lower()
    for v in ("\u02bb", "\u2019", "\u2018", "\u02bc", "`"):
        s = s.replace(v, "'")
    s = re.sub(r"[^a-z0-9@' ]+", " ", s)
    return " ".join(s.split())


def tfidf_lr(C=5.0, class_weight=None):
    """Bazaviy quvur: normallashtirish -> TF-IDF (1-2 gram) -> LogisticRegression."""
    return make_pipeline(
        TfidfVectorizer(preprocessor=normallashtir, token_pattern=r"[^ ]+",
                        ngram_range=(1, 2), sublinear_tf=True, min_df=2),
        LogisticRegression(C=C, max_iter=3000, class_weight=class_weight))


def makro_f1(y, p):
    return f1_score(y, p, average="macro")


def juft_farq(a, b):
    """Juftlashgan farq: o'rtacha va SE (a, b - bir xil misollar/foldlar)."""
    d = np.asarray(a, float) - np.asarray(b, float)
    return d.mean(), d.std(ddof=1) / np.sqrt(len(d))


def main() -> None:
    matn, y, muallif, mavzu, takror = korpus(seed=0)
    ok = ~takror
    matn, y, muallif, mavzu = matn[ok], y[ok], muallif[ok], mavzu[ok]
    gss = GroupShuffleSplit(n_splits=1, test_size=0.25, random_state=0)
    tr, te = next(gss.split(matn, y, groups=muallif))   # muallif bo'yicha bo'lish
    yt = y[te]

    print("=== 1. Sinflar nomutanosib ===")
    for k, s in enumerate(SINFLAR):
        print(f"  {s:<8} o'quv {np.mean(y[tr] == k):.1%}, test {np.mean(yt == k):.1%}")

    print("\n=== 2. Aniqlik va makro-F1: noto'g'ri va to'g'ri o'lchov ===")
    bash = {"ko'pchilik (ijobiy)": np.full(len(te), 2),
            "TF-IDF": tfidf_lr().fit(matn[tr], y[tr]).predict(matn[te]),
            "TF-IDF, balanced": tfidf_lr(class_weight="balanced")
                                .fit(matn[tr], y[tr]).predict(matn[te])}
    print(f"  {'model':<20} {'aniqlik':>8} {'makro-F1':>9}   recall: "
          + " ".join(f"{s:>7}" for s in SINFLAR))
    for nom, p in bash.items():
        rec = [np.mean(p[yt == k] == k) for k in range(3)]
        print(f"  {nom:<20} {np.mean(p == yt):>8.3f} {makro_f1(yt, p):>9.3f}   "
              f"        " + " ".join(f"{r:>7.3f}" for r in rec))
    p0, p1 = bash["TF-IDF"], bash["TF-IDF, balanced"]
    for nom, m in [("hamma misollar", np.ones(len(te), bool)), ("neytral", yt == 1)]:
        o, se = juft_farq(p1[m] == yt[m], p0[m] == yt[m])
        print(f"  balanced - oddiy, {nom:<15} {o:+.3f} (2*SE {2 * se:.3f}) -> "
              + ("sezilarli" if abs(o) > 2 * se else "sezilmas"))

    tanlangan = max(["TF-IDF", "TF-IDF, balanced"],
                    key=lambda k: makro_f1(yt, bash[k]))
    print(f"  makro-F1 bo'yicha yaxshirog'i: {tanlangan}")

    print("\n=== 3. Xato tahlili jadvali (makro-F1 bo'yicha yaxshirog'i) ===")
    xato = bash[tanlangan] != yt
    tanish = {s for mz in MAVZULAR.values() for j, ij, sl in mz[0]
              for s in normallashtir(" ".join([j] + ij + sl)).split()}
    tanish |= {s for mz in MAVZULAR.values() for f in mz[1] for s in normallashtir(f).split()}
    tanish |= {s for r in list(UMUMIY.values()) + [SALOM, XAYR]
               for f in r for s in normallashtir(f).split()}
    tokenlar = [normallashtir(m).split() for m in matn[te]]
    belgilar = {
        "uzunlik": np.array(["qisqa (<=6)" if len(t) <= 6 else
                             "o'rta (7-10)" if len(t) <= 10 else "uzun (>10)"
                             for t in tokenlar]),
        "inkor (emas)": np.array(["bor" if "emas" in t else "yo'q" for t in tokenlar]),
        "imlo xatosi": np.array(["bor" if any(s not in tanish and not s.startswith("@")
                                              for s in t) else "yo'q"
                                 for t in tokenlar]),
        "apostrof": np.array(["nostandart" if re.search("[\u02bb\u2019`]", m)
                              else "standart" for m in matn[te]]),
        "mavzu": mavzu[te],
        "haqiqiy sinf": np.array([SINFLAR[k] for k in yt]),
    }
    print(f"  umumiy xato ulushi: {xato.mean():.3f} ({xato.sum()} / {len(te)})")
    print(f"  {'belgi':<14} {'qiymat':<13} {'n':>5} {'xato':>6} {'2*SE':>6}")
    eng = ("", "", 0.0)
    for nom, qiymat in belgilar.items():
        for q in sorted(set(qiymat.tolist())):
            m = qiymat == q
            e = xato[m].mean()
            se = np.sqrt(e * (1 - e) / m.sum())
            print(f"  {nom:<14} {q:<13} {m.sum():>5} {e:>6.3f} {2 * se:>6.3f}")
            if m.sum() >= 30 and e > eng[2]:
                eng = (nom, q, e)
    print(f"  eng xavfli segment (n>=30): {eng[0]} = {eng[1]}, xato {eng[2]:.3f}")


if __name__ == "__main__":
    main()

=== 1. Sinflar nomutanosib ===
  salbiy   o'quv 26.1%, test 21.6%
  neytral  o'quv 14.1%, test 13.0%
  ijobiy   o'quv 59.9%, test 65.4%

=== 2. Aniqlik va makro-F1: noto'g'ri va to'g'ri o'lchov ===
  model                 aniqlik  makro-F1   recall:  salbiy neytral  ijobiy
  ko'pchilik (ijobiy)     0.654     0.263             0.000   0.000   1.000
  TF-IDF                  0.797     0.743             0.523   0.713   0.904
  TF-IDF, balanced        0.758     0.719             0.626   0.775   0.798
  balanced - oddiy, hamma misollar  -0.039 (2*SE 0.020) -> sezilarli
  balanced - oddiy, neytral         +0.062 (2*SE 0.043) -> sezilarli
  makro-F1 bo'yicha yaxshirog'i: TF-IDF

=== 3. Xato tahlili jadvali (makro-F1 bo'yicha yaxshirog'i) ===
  umumiy xato ulushi: 0.203 (201 / 990)
  belgi          qiymat            n   xato   2*SE
  uzunlik        o'rta (7-10)    565  0.204  0.034
  uzunlik        qisqa (<=6)     286  0.171  0.045
  uzunlik        uzun (>10)      139  0.266  0.075
  inkor (emas)   bor             214  0.215  0.056
  inkor (emas)   yo'q            776  0.200  0.029
  imlo xatosi    bor             368  0.228  0.044
  imlo xatosi    yo'q            622  0.188  0.031
  apostrof       nostandart      321  0.218  0.046
  apostrof       standart        669  0.196  0.031
  mavzu          kiyim           296  0.216  0.048
  mavzu          mehmonxona      300  0.217  0.048
  mavzu          restoran        267  0.180  0.047
  mavzu          telefon         127  0.189  0.069
  haqiqiy sinf   ijobiy          647  0.096  0.023
  haqiqiy sinf   neytral         129  0.287  0.080
  haqiqiy sinf   salbiy          214  0.477  0.068
  eng xavfli segment (n>=30): haqiqiy sinf = salbiy, xato 0.477

train_test_split(matn, y, test_size=0.2)                        # ⚠️ nusxalar ikki tomonda
next(GroupShuffleSplit(test_size=0.2).split(matn, y, groups=kl))  # ✅ klaster bir tomonda

takror = pd.Series(matn).duplicated()                           # ⚠️ "Xonasi toza!" != "xonasi toza"
takror = pd.Series([xesh(m) for m in matn]).duplicated()        # ✅ normallashtirilgan xesh

X = TfidfVectorizer().fit_transform(matn); cross_val_score(lr, X, y)   # ⚠️
cross_val_score(make_pipeline(TfidfVectorizer(), lr), matn, y)          # ✅

X = SelectKBest(chi2, k=300).fit_transform(X, y)                # ⚠️ test yorliqlari ko'rildi
make_pipeline(TfidfVectorizer(), SelectKBest(chi2, k=300), lr)  # ✅

KFold(5, shuffle=True).split(matn)                              # ⚠️ "@taxallus" -> sinf
GroupKFold(5).split(matn, y, groups=muallif)                    # ✅

print(np.mean(p == y))                                          # ⚠️ "doim ijobiy" = 0.65
print(f1_score(y, p, average="macro"), recall_sinf_boyicha)     # ✅

print(xato[imlo].mean(), xato[~imlo].mean())                    # ⚠️ 0.228 > 0.188 - "topilma"?
e = xato[m].mean(); print(e, 2 * np.sqrt(e * (1 - e) / m.sum()))  # ✅ n va 2*SE bilan

1.  # "Xonasi toza!" va "xonasi toza" - xesh bir xil bo'ladimi?
2.  # takror klasterini qaysi bo'lish sinfi bilan ishlatish mumkin?
3.  # kosinus 0.95 bo'lgan ikki matn ma'nosi teskari bo'lishi mumkinmi?
4.  # vektorizatorni butun ma'lumotda fit qilish nimani sizdiradi?
5.  # aralashtirilgan yorliqlarda to'g'ri CV qanday baho berishi kerak?
6.  # SelectKBest(chi2) ni tashqarida fit qilish nega vektorizatordan xavfliroq?
7.  # taxallus "@..." belgisi GroupKFold da nima bo'ladi?
8.  # KFold va GroupKFold natijalarini juftlashgan farq bilan solishtirish mumkinmi?
9.  # OOV ulushi 8% dan 52% ga oshdi - bu nimani bildiradi?
10. # "doim ijobiy" modelining 3 sinfli makro-F1 i taxminan?
11. # class_weight="balanced" makro-F1 ni doim oshiradimi?
12. # xato jadvalida n=20 li segmentda xato 0.5 - xulosa qilasizmi?

1.  tr, te = train_test_split(np.arange(len(y)), test_size=0.2)   # 24% takror bor

2.  X = TfidfVectorizer().fit_transform(matn)
    print(cross_val_score(LogisticRegression(), X, y).mean())

3.  X = SelectKBest(chi2, k=300).fit_transform(X, y)
    print(cross_val_score(LogisticRegression(), X, y).mean())

4.  print(cross_val_score(quvur, matn, y, cv=KFold(5, shuffle=True)).mean())
    # har muallifda 8-60 ta sharh

5.  print("imlo xatoli sharhlarda xato ko'p:", xato[imlo].mean() > xato[~imlo].mean())