27.4-dars: Eksperiment kuzatuvi

model_final.pkl               "LogReg, AUC 0.745"
model_final_v2.pkl            "GB, AUC 0.780"
model_final_v3_haqiqiy.pkl    "GB lr=0.3, AUC 0.944 - ENG YAXSHI!"

MUAMMOLAR:
  1. USTIGA YOZISH  katak qayta ishga tushdi -> model_final.pkl endi BOSHQA model
  2. SOLISHTIRIB BO'LMAYDI  har AUC boshqa validatsiyada / boshqa ma'lumotda
  3. KONTEKST YO'Q  qaysi kod? qaysi belgilar? qaysi urug'? qaysi versiya?
  4. SO'ROV YO'Q  "depth=3 dagi eng yaxshisi" - qo'lda qidirish
  5. MUVAFFAQIYATSIZLAR YO'Q  faqat "yaxshi" natijalar yoziladi
     -> nechta variant sinalgani noma'lum -> optimizmni baholab bo'lmaydi
  6. NOM - MA'LUMOT EMAS  "haqiqiy", "final", "yangi" - hech narsa demaydi

QOIDA: yozuv shunchalik to'liq bo'lsinki, boshqa odam UNDAN
  1) modelni qayta qura olsin
  2) natijani qaysi sharoitda olinganini bilsin
  3) uni boshqa yurish bilan solishtirish mumkinmi - aniqlay olsin

EKSPERIMENT  (masalan, "churn_gb_qidiruv")
  +-- YURISH (run)  id, nom, holat, ma'lumot xeshi, kod xeshi
        +-- PARAM     kalit -> qiymat (bir marta, o'zgarmas)
        +-- METRIKA   kalit, QADAM -> qiymat (ko'p marta)
        +-- TEG       kalit -> qiymat (o'zgarishi mumkin)
        +-- ARTEFAKT  fayl yo'li + sha256

SAQLASH (MLflow kabi tizimlarda):
  metama'lumot omborchasi (backend store): sqlite / PostgreSQL
    -> param, metrika, teg: kichik, ko'p, SO'ROV qilinadi
  artefakt omborchasi (artifact store): fayl tizimi / S3 / GCS
    -> model, grafik: katta, kam, faqat yuklab olinadi

MLflow FAYL REJIMI (mlruns/):
  mlruns/<eksperiment_id>/<run_id>/
      meta.yaml, params/<kalit>, metrics/<kalit>, tags/<kalit>, artifacts/

SO'ROVLAR:
  eng yaxshi K yurish:  WHERE holat='FINISHED' AND kalit='val_auc'
                        ORDER BY qiymat DESC LIMIT K
  filtr:                param max_depth = 3, teg model = 'GB'
  erta to'xtash nuqtasi: metrika qadamlari ichida maksimum

IKKI YURISHNI SOLISHTIRISH:
  1. SHART: bir xil validatsiya xeshi (aks holda - to'xtash)
  2. parametr farqi: faqat farq qiladigan kalitlar
  3. metrikalar qadamma-qadam (egri chiziqlar)
  4. farqning NOANIQLIGI: bir xil validatsiya qatorlarida juftlashgan
     bootstrap -> farq va SE; |farq| > 2*SE -> sezilarli
     (buning uchun artefakt - model yoki bashoratlar - kerak!)

K ta yurish, har birining validatsiya bahosi = haqiqiy sifat + shovqin
  val_k = haqiqiy_k + e_k,   e_k ~ validatsiya to'plami tasodifi

max_k(val_k) ni tanlaymiz:
  haqiqiy sifatlar yaqin bo'lsa -> tanlov ASOSAN shovqinni tanlaydi
  tanlanganning val bahosi: haqiqiy + (eng katta e_k) -> OPTIMISTIK
  K oshgani sari optimizm oshadi (ko'p "lotereya chiptasi")

O'LCHASH: tanlangan yurishni YOPIQ testda baholash
  optimizm = val(tanlangan) - test(tanlangan)

YECHIMLAR (18.4, 18.11):
  yopiq test - faqat yakunda, BIR marta
  nested CV - tanlov jarayonining o'zini baholash
  hisobotga: K (nechta yurish) va tanlanganning TEST natijasi
  katta validatsiya -> kichik shovqin -> kichik optimizm

MUAMMO: qo'lda log_param(...) - unutiladi, ba'zi parametrlar yozilmaydi
AUTOLOG: model.get_params() dan HAMMA parametr, ma'lumot xeshi,
         kod xeshi, versiya, metrika, artefakt - avtomatik
         (mlflow.autolog() - sklearn, xgboost, pytorch uchun)

QAYTA TIKLASH (yozuvdan):
  1. ma'lumot xeshi mosmi? -> yo'q bo'lsa RAD ETISH
  2. model turi + parametrlar -> model qayta quriladi
  3. qayta o'qitish -> metrika yozuvdagiga TENGmi
  4. artefakt sha256 mosmi; saqlangan va qayta qurilgan bashoratlar tengmi

import mlflow
import mlflow.sklearn
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_auc_score

mlflow.set_tracking_uri("sqlite:///mlflow.db")      # yoki "http://server:5000"
mlflow.set_experiment("churn_gb_qidiruv")

with mlflow.start_run(run_name="lr0.1_d3") as run:
    params = {"learning_rate": 0.1, "max_depth": 3, "n_estimators": 100}
    mlflow.log_params(params)                        # yoki log_param(k, v)
    mlflow.set_tag("model", "GradientBoosting")
    mlflow.set_tag("malumot_xesh", "952c6655a350")

    model = GradientBoostingClassifier(random_state=0, **params).fit(X_tr, y_tr)
    for qadam, p in enumerate(model.staged_predict_proba(X_va), 1):
        if qadam % 20 == 0:
            mlflow.log_metric("val_auc", roc_auc_score(y_va, p[:, 1]), step=qadam)

    mlflow.log_artifact("konfig.json")               # istalgan fayl
    mlflow.sklearn.log_model(model, name="model")    # model + muhit + imzo

# so'rov: eng yaxshi yurishlar (pandas DataFrame qaytaradi)
eng = mlflow.search_runs(
    experiment_names=["churn_gb_qidiruv"],
    filter_string="params.max_depth = '3' and attributes.status = 'FINISHED'",
    order_by=["metrics.val_auc DESC"], max_results=5)

# yurishdan modelni yuklash
model = mlflow.sklearn.load_model(f"runs:/{run.info.run_id}/model")

# avtomatik qayd qilish
mlflow.sklearn.autolog()                             # fit() da hammasi yoziladi

import hashlib
import json
import sqlite3
from contextlib import contextmanager

SXEMA = """
CREATE TABLE IF NOT EXISTS yurish (id INTEGER PRIMARY KEY, eksperiment TEXT,
    nom TEXT, holat TEXT, malumot_xesh TEXT, kod_xesh TEXT);
CREATE TABLE IF NOT EXISTS param (yurish_id INTEGER, kalit TEXT, qiymat TEXT,
    PRIMARY KEY (yurish_id, kalit));
CREATE TABLE IF NOT EXISTS metrika (yurish_id INTEGER, kalit TEXT, qadam INTEGER,
    qiymat REAL, PRIMARY KEY (yurish_id, kalit, qadam));
"""


@contextmanager
def start_run(db, eksperiment, nom):
    with db:
        yid = db.execute("INSERT INTO yurish (eksperiment, nom, holat) "
                         "VALUES (?, ?, 'RUNNING')", (eksperiment, nom)).lastrowid
    try:
        yield yid
    except Exception:
        with db:
            db.execute("UPDATE yurish SET holat='FAILED' WHERE id=?", (yid,))
        raise
    with db:
        db.execute("UPDATE yurish SET holat='FINISHED' WHERE id=?", (yid,))


def log_param(db, yid, k, v):
    with db:
        db.execute("INSERT INTO param VALUES (?, ?, ?)", (yid, k, json.dumps(v)))


def log_metric(db, yid, k, v, qadam=0):
    with db:
        db.execute("INSERT OR REPLACE INTO metrika VALUES (?, ?, ?, ?)",
                   (yid, k, qadam, float(v)))


ENG_YAXSHI = """
SELECT y.nom, m.qiymat FROM yurish y JOIN metrika m ON m.yurish_id = y.id
WHERE y.holat = 'FINISHED' AND m.kalit = 'val_auc' AND m.qadam = ?
ORDER BY m.qiymat DESC LIMIT ?"""


def artefakt_xesh(yol):
    return hashlib.sha256(open(yol, "rb").read()).hexdigest()[:12]

yurish = param + metrika(qadam) + teg + artefakt(sha256) + holat
       + ma'lumot xeshi + kod xeshi + muhit
solishtirish sharti: bir xil validatsiya xeshi
farq: juftlashgan bootstrap SE (artefakt kerak)
g'olib la'nati: max(val) optimistik, K bilan oshadi -> yopiq test
autolog: get_params() dan hammasi; qayta tiklash - yozuv to'liqligining testi
MLflow: start_run, log_param, log_metric(step), log_artifact, log_model, search_runs

"""Nega model_final_v3_haqiqiy.pkl ishlamaydi va yurish yozuvi nima beradi."""

import hashlib
import inspect
import json
import tempfile
from importlib.metadata import version
from pathlib import Path

import joblib
import numpy as np
from sklearn.datasets import make_classification
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

X_ALL, Y_ALL = make_classification(n_samples=6000, n_features=10, n_informative=4,
                                   flip_y=0.0, random_state=1)


def shovqin(y, ulush, seed):
    """Belgilash xatolari: berilgan ulushdagi belgilar teskari."""
    rng = np.random.default_rng(seed)
    y = y.copy()
    i = rng.random(len(y)) < ulush
    y[i] = 1 - y[i]
    return y


def malumot(versiya):
    """v1: belgilash shovqinli (20%); v2: ma'lumot 'jim' tozalangan (3%)."""
    ulush = {"v1": 0.20, "v2": 0.03}[versiya]
    X, y = X_ALL[:3000], shovqin(Y_ALL[:3000], ulush, seed=5)
    return train_test_split(X, y, test_size=0.3, random_state=0)


def orgat(model, versiya):
    Xtr, Xva, ytr, yva = malumot(versiya)
    model.fit(Xtr, ytr)
    return model, roc_auc_score(yva, model.predict_proba(Xva)[:, 1])


def xesh(b):
    return hashlib.sha256(b).hexdigest()[:12]


def auc_farq_se(y, p1, p2, n_boot=300, seed=0):
    """Bir xil test qatorlarida juftlashgan bootstrap: AUC farqi va SE."""
    rng = np.random.default_rng(seed)
    farqlar = []
    for _ in range(n_boot):
        i = rng.integers(0, len(y), len(y))
        farqlar.append(roc_auc_score(y[i], p1[i]) - roc_auc_score(y[i], p2[i]))
    return roc_auc_score(y, p1) - roc_auc_score(y, p2), float(np.std(farqlar, ddof=1))


def main() -> None:
    with tempfile.TemporaryDirectory() as t:
        papka = Path(t)
        eslatma, eslatma_auc = {}, {}

        print("=== 1. Noutbuk sessiyasi: fayllar va README eslatmalari ===")
        m, a = orgat(LogisticRegression(max_iter=1000), "v1")
        joblib.dump(m, papka / "model_final.pkl")
        eslatma["model_final.pkl"], eslatma_auc["model_final.pkl"] = "LogReg", a
        m, a = orgat(GradientBoostingClassifier(random_state=0), "v1")
        joblib.dump(m, papka / "model_final_v2.pkl")
        eslatma["model_final_v2.pkl"], eslatma_auc["model_final_v2.pkl"] = "GB", a
        m, a = orgat(RandomForestClassifier(n_estimators=100, random_state=0), "v2")
        joblib.dump(m, papka / "model_final.pkl")          # katak qayta ishga tushdi!
        m, a = orgat(GradientBoostingClassifier(learning_rate=0.3, random_state=0), "v2")
        joblib.dump(m, papka / "model_final_v3_haqiqiy.pkl")
        eslatma["model_final_v3_haqiqiy.pkl"] = "GB lr=0.3 - ENG YAXSHI!"
        eslatma_auc["model_final_v3_haqiqiy.pkl"] = a
        for f in sorted(papka.glob("*.pkl")):
            print(f"  {f.name:<28} README: {eslatma[f.name]}, "
                  f"AUC {eslatma_auc[f.name]:.3f}")

        print("\n=== 2. Savollarga javob berishga urinish ===")
        for f in sorted(papka.glob("*.pkl")):
            print(f"  {f.name:<28} haqiqiy turi: {type(joblib.load(f)).__name__}")
        print("  model_final.pkl: README da LogReg, faylda RandomForest (ustiga yozilgan)")
        print("  qaysi ma'lumot? qaysi kod? qaysi validatsiya? - NOMA'LUM")

        print("\n=== 3. Bir xil yangi test to'plamida qayta baholash ===")
        Xt, yt = X_ALL[3000:], shovqin(Y_ALL[3000:], 0.03, seed=6)
        pr = {}
        for f in sorted(papka.glob("*.pkl")):
            pr[f.name] = joblib.load(f).predict_proba(Xt)[:, 1]
            print(f"  {f.name:<28} test AUC {roc_auc_score(yt, pr[f.name]):.4f}")
        v3, v2 = "model_final_v3_haqiqiy.pkl", "model_final_v2.pkl"
        d, se = auc_farq_se(yt, pr[v3], pr[v2])
        print(f"  v3 - v2: README da {eslatma_auc[v3] - eslatma_auc[v2]:+.3f}, "
              f"bir xil testda {d:+.4f} (SE {se:.4f})")

        print("\n=== 4. To'g'ri yo'l: har yurish - to'liq yozuv ===")
        kod_xesh = xesh(inspect.getsource(orgat).encode())
        yozuvlar = []
        for nom, model, v in [
                ("logreg_v1", LogisticRegression(max_iter=1000), "v1"),
                ("gb_v1", GradientBoostingClassifier(random_state=0), "v1"),
                ("gb_v2", GradientBoostingClassifier(random_state=0), "v2"),
                ("gb_lr03_v2", GradientBoostingClassifier(learning_rate=0.3,
                                                          random_state=0), "v2")]:
            m, a = orgat(model, v)
            Xtr, Xva, ytr, yva = malumot(v)
            yol = papka / f"{nom}.joblib"
            joblib.dump(m, yol)
            yozuvlar.append({
                "yurish": nom, "model": type(m).__name__,
                "param": {k: m.get_params()[k] for k in sorted(m.get_params())
                          if k in ("C", "learning_rate", "n_estimators", "max_depth")},
                "metrika": {"val_auc": round(a, 4)},
                "malumot": {"versiya": v, "xesh": xesh(Xtr.tobytes() + ytr.tobytes()),
                            "val_xesh": xesh(Xva.tobytes() + yva.tobytes())},
                "kod_xesh": kod_xesh, "sklearn": version("scikit-learn"),
                "artefakt": {"yol": yol.name, "sha256": xesh(yol.read_bytes())}})
        (papka / "yurishlar.jsonl").write_text(
            "\n".join(json.dumps(y) for y in yozuvlar) + "\n", encoding="utf-8")
        print(f"  bitta yozuv kalitlari: {list(yozuvlar[0])}")
        for y in yozuvlar:
            print(f"  {y['yurish']:<11} val_auc {y['metrika']['val_auc']:.4f}  "
                  f"val_xesh {y['malumot']['val_xesh']}")
        guruh = {}
        for y in yozuvlar:
            guruh.setdefault(y["malumot"]["val_xesh"], []).append(y)
        print("  faqat bir xil validatsiya ichida solishtiramiz:")
        for g in sorted(guruh.values(), key=lambda g: g[0]["yurish"]):
            a, b = g[0], g[-1]
            print(f"    {b['yurish']} - {a['yurish']}: "
                  f"{b['metrika']['val_auc'] - a['metrika']['val_auc']:+.4f}")
        print(f"  gb_v2 - gb_v1 (bir xil model, faqat MA'LUMOT boshqa): "
              f"{yozuvlar[2]['metrika']['val_auc'] - yozuvlar[1]['metrika']['val_auc']:+.4f}"
              f" - bu model yutug'i EMAS")


if __name__ == "__main__":
    main()

=== 1. Noutbuk sessiyasi: fayllar va README eslatmalari ===
  model_final.pkl              README: LogReg, AUC 0.745
  model_final_v2.pkl           README: GB, AUC 0.780
  model_final_v3_haqiqiy.pkl   README: GB lr=0.3 - ENG YAXSHI!, AUC 0.944

=== 2. Savollarga javob berishga urinish ===
  model_final.pkl              haqiqiy turi: RandomForestClassifier
  model_final_v2.pkl           haqiqiy turi: GradientBoostingClassifier
  model_final_v3_haqiqiy.pkl   haqiqiy turi: GradientBoostingClassifier
  model_final.pkl: README da LogReg, faylda RandomForest (ustiga yozilgan)
  qaysi ma'lumot? qaysi kod? qaysi validatsiya? - NOMA'LUM

=== 3. Bir xil yangi test to'plamida qayta baholash ===
  model_final.pkl              test AUC 0.9516
  model_final_v2.pkl           test AUC 0.9274
  model_final_v3_haqiqiy.pkl   test AUC 0.9466
  v3 - v2: README da +0.164, bir xil testda +0.0192 (SE 0.0034)

=== 4. To'g'ri yo'l: har yurish - to'liq yozuv ===
  bitta yozuv kalitlari: ['yurish', 'model', 'param', 'metrika', 'malumot', 'kod_xesh', 'sklearn', 'artefakt']
  logreg_v1   val_auc 0.7448  val_xesh 7935fc4f1ad8
  gb_v1       val_auc 0.7805  val_xesh 7935fc4f1ad8
  gb_v2       val_auc 0.9502  val_xesh 92a7aea2fafe
  gb_lr03_v2  val_auc 0.9441  val_xesh 92a7aea2fafe
  faqat bir xil validatsiya ichida solishtiramiz:
    gb_lr03_v2 - gb_v2: -0.0061
    gb_v1 - logreg_v1: +0.0357
  gb_v2 - gb_v1 (bir xil model, faqat MA'LUMOT boshqa): +0.1697 - bu model yutug'i EMAS

"""sqlite ustida MLflow ga o'xshash kichik eksperiment kuzatuvchisi."""

import hashlib
import json
import sqlite3
import tempfile
from contextlib import closing, contextmanager
from pathlib import Path

import joblib
import numpy as np
from sklearn.datasets import make_classification
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

SXEMA = """
CREATE TABLE IF NOT EXISTS yurish (id INTEGER PRIMARY KEY, eksperiment TEXT,
    nom TEXT, holat TEXT, malumot_xesh TEXT, kod_xesh TEXT);
CREATE TABLE IF NOT EXISTS param (yurish_id INTEGER, kalit TEXT, qiymat TEXT,
    PRIMARY KEY (yurish_id, kalit));
CREATE TABLE IF NOT EXISTS metrika (yurish_id INTEGER, kalit TEXT, qadam INTEGER,
    qiymat REAL, PRIMARY KEY (yurish_id, kalit, qadam));
CREATE TABLE IF NOT EXISTS teg (yurish_id INTEGER, kalit TEXT, qiymat TEXT,
    PRIMARY KEY (yurish_id, kalit));
CREATE TABLE IF NOT EXISTS artefakt (yurish_id INTEGER, yol TEXT, sha256 TEXT);
"""


class Yurish:
    def __init__(self, tracker, yid):
        self.t, self.id = tracker, yid

    def log_param(self, kalit, qiymat):          # PRIMARY KEY: qayta yozib bo'lmaydi
        with self.t.db:
            self.t.db.execute("INSERT INTO param VALUES (?, ?, ?)",
                              (self.id, kalit, json.dumps(qiymat)))

    def log_metric(self, kalit, qiymat, qadam=0):
        with self.t.db:
            self.t.db.execute("INSERT OR REPLACE INTO metrika VALUES (?, ?, ?, ?)",
                              (self.id, kalit, qadam, float(qiymat)))

    def set_tag(self, kalit, qiymat):
        with self.t.db:
            self.t.db.execute("INSERT OR REPLACE INTO teg VALUES (?, ?, ?)",
                              (self.id, kalit, str(qiymat)))

    def log_artifact(self, manba, nom):
        yol = self.t.papka / "artefaktlar" / str(self.id) / nom
        yol.parent.mkdir(parents=True, exist_ok=True)
        yol.write_bytes(Path(manba).read_bytes())
        sha = hashlib.sha256(yol.read_bytes()).hexdigest()[:12]
        with self.t.db:
            self.t.db.execute("INSERT INTO artefakt VALUES (?, ?, ?)",
                              (self.id, yol.relative_to(self.t.papka).as_posix(), sha))


class Tracker:
    def __init__(self, papka):
        self.papka = Path(papka)
        self.db = sqlite3.connect(self.papka / "kuzatuv.db")
        self.db.executescript(SXEMA)

    def close(self):
        self.db.close()

    @contextmanager
    def start_run(self, eksperiment, nom, malumot_xesh="", kod_xesh=""):
        with self.db:
            cur = self.db.execute(
                "INSERT INTO yurish (eksperiment, nom, holat, malumot_xesh, kod_xesh) "
                "VALUES (?, ?, 'RUNNING', ?, ?)",
                (eksperiment, nom, malumot_xesh, kod_xesh))
        y = Yurish(self, cur.lastrowid)
        try:
            yield y
        except Exception as xato:
            y.set_tag("xato", f"{type(xato).__name__}: {xato}")
            self._holat(y.id, "FAILED")
            raise
        self._holat(y.id, "FINISHED")

    def _holat(self, yid, holat):
        with self.db:
            self.db.execute("UPDATE yurish SET holat=? WHERE id=?", (holat, yid))

    def yurish_ol(self, nom):
        yid, mx, kx = self.db.execute("SELECT id, malumot_xesh, kod_xesh FROM yurish "
                                      "WHERE nom=?", (nom,)).fetchone()
        return {"id": yid, "malumot": mx, "kod": kx,
                "param": dict(self.db.execute(
                    "SELECT kalit, qiymat FROM param WHERE yurish_id=?", (yid,))),
                "val_auc": dict(self.db.execute(
                    "SELECT qadam, qiymat FROM metrika WHERE yurish_id=? "
                    "AND kalit='val_auc'", (yid,))),
                "artefakt": self.db.execute(
                    "SELECT yol, sha256 FROM artefakt WHERE yurish_id=?",
                    (yid,)).fetchone()}


def main() -> None:
    X, y = make_classification(n_samples=2500, n_features=12, n_informative=5,
                               flip_y=0.05, random_state=0)
    Xtr, Xva, ytr, yva = train_test_split(X, y, test_size=0.3, random_state=0,
                                          stratify=y)
    mxesh = hashlib.sha256(Xva.tobytes() + yva.tobytes()).hexdigest()[:12]
    kxesh = "a1b2c3d"                          # haqiqiy loyihada: git rev-parse HEAD

    with tempfile.TemporaryDirectory() as t, closing(Tracker(t)) as tr:
        print("=== 1. Giperparametr qidiruvi - har kombinatsiya bitta yurish ===")
        tarmoq = [{"learning_rate": lr, "max_depth": d, "n_estimators": 100}
                  for lr in [0.03, 0.1, 0.3] for d in [2, 3, 5]]
        tarmoq.append({"learning_rate": -0.1, "max_depth": 3, "n_estimators": 100})
        for i, p in enumerate(tarmoq, 1):
            try:
                with tr.start_run("gb_qidiruv", f"yurish_{i:02d}", mxesh, kxesh) as r:
                    for k, v in p.items():
                        r.log_param(k, v)
                    r.set_tag("model", "GradientBoosting")
                    m = GradientBoostingClassifier(random_state=0, **p).fit(Xtr, ytr)
                    for qadam, pr in enumerate(m.staged_predict_proba(Xva), 1):
                        if qadam % 20 == 0:
                            r.log_metric("val_auc", roc_auc_score(yva, pr[:, 1]), qadam)
                    joblib.dump(m, Path(t) / "model.joblib")
                    r.log_artifact(Path(t) / "model.joblib", "model.joblib")
            except Exception:
                pass                           # yurish FAILED sifatida yozildi
        holatlar = tr.db.execute("SELECT holat, COUNT(*) FROM yurish GROUP BY holat "
                                 "ORDER BY holat").fetchall()
        print(f"  yurishlar: {holatlar}")
        nom, xato = tr.db.execute("SELECT y.nom, t.qiymat FROM yurish y JOIN teg t "
                                  "ON t.yurish_id = y.id WHERE t.kalit='xato'").fetchone()
        print(f"  muvaffaqiyatsiz: {nom} -> {xato[:58]}")
        for jadval in ["param", "metrika", "teg", "artefakt"]:
            n = tr.db.execute(f"SELECT COUNT(*) FROM {jadval}").fetchone()[0]
            print(f"  {jadval:<9} {n} qator")

        print("\n=== 2. So'rov: eng yaxshi 3 yurish (100-qadamdagi val_auc) ===")
        sorov = """
            SELECT y.nom, m.qiymat,
                   (SELECT qiymat FROM param WHERE yurish_id=y.id AND kalit='learning_rate'),
                   (SELECT qiymat FROM param WHERE yurish_id=y.id AND kalit='max_depth')
            FROM yurish y JOIN metrika m ON m.yurish_id = y.id
            WHERE y.holat='FINISHED' AND m.kalit='val_auc' AND m.qadam=100
            ORDER BY m.qiymat DESC LIMIT 3"""
        eng3 = tr.db.execute(sorov).fetchall()
        for nom, auc, lr, d in eng3:
            print(f"  {nom}: val_auc {auc:.4f}  lr={lr} depth={d}")
        filtr = tr.db.execute("""
            SELECT y.nom, m.qiymat FROM yurish y
            JOIN param p ON p.yurish_id = y.id AND p.kalit='max_depth' AND p.qiymat='2'
            JOIN metrika m ON m.yurish_id = y.id AND m.kalit='val_auc' AND m.qadam=100
            ORDER BY m.qiymat DESC""").fetchall()
        print(f"  filtr max_depth=2: {[(n, round(v, 4)) for n, v in filtr]}")
        nom, qadam, auc = tr.db.execute("""
            SELECT y.nom, m.qadam, m.qiymat FROM metrika m JOIN yurish y
            ON y.id = m.yurish_id WHERE m.kalit='val_auc'
            ORDER BY m.qiymat DESC LIMIT 1""").fetchone()
        print(f"  barcha qadamlar ichida eng yuqori: {nom}, qadam {qadam}: {auc:.4f}")

        print("\n=== 3. Eng yaxshi ikki yurishni solishtirish ===")
        a, b = tr.yurish_ol(eng3[0][0]), tr.yurish_ol(eng3[1][0])
        print(f"  bir xil ma'lumot va kod: "
              f"{a['malumot'] == b['malumot'] and a['kod'] == b['kod']}")
        farq = {k: (a["param"][k], b["param"][k]) for k in sorted(a["param"])
                if a["param"][k] != b["param"][k]}
        print(f"  parametr farqi ({eng3[0][0]}, {eng3[1][0]}): {farq}")
        print(f"  {'qadam':>5} {eng3[0][0]:>10} {eng3[1][0]:>10}")
        for q in sorted(a["val_auc"]):
            print(f"  {q:>5} {a['val_auc'][q]:>10.4f} {b['val_auc'][q]:>10.4f}")
        pa, pb = [joblib.load(Path(t) / r["artefakt"][0]).predict_proba(Xva)[:, 1]
                  for r in (a, b)]
        rng = np.random.default_rng(0)
        boot = []
        for _ in range(300):
            i = rng.integers(0, len(yva), len(yva))
            boot.append(roc_auc_score(yva[i], pa[i]) - roc_auc_score(yva[i], pb[i]))
        d = roc_auc_score(yva, pa) - roc_auc_score(yva, pb)
        se = float(np.std(boot, ddof=1))
        print(f"  farq {d:+.4f}, juftlashgan bootstrap SE {se:.4f} -> "
              f"{'sezilarli' if abs(d) > 2 * se else 'sezilarli emas'}")


if __name__ == "__main__":
    main()

=== 1. Giperparametr qidiruvi - har kombinatsiya bitta yurish ===
  yurishlar: [('FAILED', 1), ('FINISHED', 9)]
  muvaffaqiyatsiz: yurish_10 -> InvalidParameterError: The 'learning_rate' parameter of Gr
  param     30 qator
  metrika   45 qator
  teg       11 qator
  artefakt  9 qator

=== 2. So'rov: eng yaxshi 3 yurish (100-qadamdagi val_auc) ===
  yurish_05: val_auc 0.9445  lr=0.1 depth=3
  yurish_06: val_auc 0.9429  lr=0.1 depth=5
  yurish_03: val_auc 0.9418  lr=0.03 depth=5
  filtr max_depth=2: [('yurish_07', 0.934), ('yurish_04', 0.9303), ('yurish_01', 0.9136)]
  barcha qadamlar ichida eng yuqori: yurish_05, qadam 80: 0.9448

=== 3. Eng yaxshi ikki yurishni solishtirish ===
  bir xil ma'lumot va kod: True
  parametr farqi (yurish_05, yurish_06): {'max_depth': ('3', '5')}
  qadam  yurish_05  yurish_06
     20     0.9272     0.9368
     40     0.9365     0.9426
     60     0.9420     0.9430
     80     0.9448     0.9427
    100     0.9445     0.9429
  farq +0.0016, juftlashgan bootstrap SE 0.0031 -> sezilarli emas

"""Fayl asosidagi kuzatuvchi (mlruns uslubi), autolog va yurishdan qayta tiklash."""

import hashlib
import inspect
import json
import tempfile
from contextlib import contextmanager
from importlib.metadata import version
from pathlib import Path

import joblib
import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier, RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score

MODEL_TURLARI = {c.__name__: c for c in [LogisticRegression, RandomForestClassifier,
                                          HistGradientBoostingClassifier]}


def xesh(b):
    return hashlib.sha256(b).hexdigest()[:12]


class Yurish:
    def __init__(self, papka):
        self.papka = papka

    def log_param(self, k, v):
        (self.papka / "params" / k).write_text(json.dumps(v), encoding="utf-8")

    def log_metric(self, k, v, qadam=0):
        with open(self.papka / "metrics" / k, "a", encoding="utf-8") as f:
            f.write(f"{qadam} {float(v)!r}\n")          # MLflow: "vaqt qiymat qadam"

    def set_tag(self, k, v):
        (self.papka / "tags" / k).write_text(str(v), encoding="utf-8")

    def log_artifact(self, obj, nom):
        yol = self.papka / "artifacts" / nom
        joblib.dump(obj, yol)
        self.set_tag(f"sha256.{nom}", xesh(yol.read_bytes()))


class FaylTracker:
    """mlruns/<eksperiment>/<run_id>/{meta.json, params/, metrics/, tags/, artifacts/}"""

    def __init__(self, ildiz):
        self.ildiz = Path(ildiz) / "mlruns"
        self.hisob = 0

    @contextmanager
    def start_run(self, eksperiment, nom):
        self.hisob += 1
        rid = xesh(f"{eksperiment}/{nom}/{self.hisob}".encode())[:8]
        papka = self.ildiz / eksperiment / rid
        for p in ["params", "metrics", "tags", "artifacts"]:
            (papka / p).mkdir(parents=True)
        meta = {"run_id": rid, "nom": nom, "holat": "RUNNING"}
        try:
            yield Yurish(papka)
            meta["holat"] = "FINISHED"
        except Exception:
            meta["holat"] = "FAILED"
            raise
        finally:
            (papka / "meta.json").write_text(json.dumps(meta), encoding="utf-8")

    def search_runs(self, eksperiment, shart=lambda r: True, kalit="val_auc"):
        yurishlar = []
        for papka in sorted((self.ildiz / eksperiment).iterdir()):
            r = json.loads((papka / "meta.json").read_text(encoding="utf-8"))
            r["params"] = {f.name: json.loads(f.read_text(encoding="utf-8"))
                           for f in sorted((papka / "params").iterdir())}
            r["tags"] = {f.name: f.read_text(encoding="utf-8")
                         for f in sorted((papka / "tags").iterdir())}
            r["metrics"] = {f.name: float(f.read_text(encoding="utf-8").split()[-1])
                            for f in sorted((papka / "metrics").iterdir())}
            r["papka"] = papka
            if r["holat"] == "FINISHED" and shart(r):
                yurishlar.append(r)
        return sorted(yurishlar, key=lambda r: -r["metrics"].get(kalit, -1))


def autolog(yurish, model, Xtr, ytr, Xva, yva):
    """Qo'lda unutib qo'ymaslik uchun: hammasini avtomatik yozadi."""
    yurish.set_tag("model_turi", type(model).__name__)
    for k, v in sorted(model.get_params().items()):
        if isinstance(v, (int, float, str, bool)) or v is None:
            yurish.log_param(k, v)
    yurish.set_tag("malumot.oquv", xesh(Xtr.tobytes() + ytr.tobytes()))
    yurish.set_tag("malumot.val", xesh(Xva.tobytes() + yva.tobytes()))
    yurish.set_tag("kod", xesh(inspect.getsource(autolog).encode()))
    yurish.set_tag("sklearn", version("scikit-learn"))
    model.fit(Xtr, ytr)
    yurish.log_metric("val_auc", roc_auc_score(yva, model.predict_proba(Xva)[:, 1]))
    yurish.log_artifact(model, "model.joblib")
    return model


def qayta_tiklash(r, Xtr, ytr, Xva, yva):
    """Yurish yozuvidan modelni qayta qurish va natijani tekshirish."""
    if xesh(Xtr.tobytes() + ytr.tobytes()) != r["tags"]["malumot.oquv"]:
        raise ValueError("o'quv ma'lumoti xeshi mos emas - qayta tiklash mumkin emas")
    model = MODEL_TURLARI[r["tags"]["model_turi"]](**r["params"])
    model.fit(Xtr, ytr)
    auc = roc_auc_score(yva, model.predict_proba(Xva)[:, 1])
    yol = r["papka"] / "artifacts" / "model.joblib"
    sha_mos = xesh(yol.read_bytes()) == r["tags"]["sha256.model.joblib"]
    saqlangan = joblib.load(yol)
    bir_xil = np.array_equal(saqlangan.predict_proba(Xva), model.predict_proba(Xva))
    return auc, sha_mos, bir_xil


def main() -> None:
    rng = np.random.default_rng(0)
    X = rng.normal(size=(3000, 8))
    y = (rng.random(3000) < 1 / (1 + np.exp(-(X[:, 0] - X[:, 1] * X[:, 2]
                                               + 0.5 * X[:, 3])))).astype(int)
    Xtr, ytr, Xva, yva = X[:2000], y[:2000], X[2000:], y[2000:]

    with tempfile.TemporaryDirectory() as t:
        tr = FaylTracker(t)
        print("=== 1. Autolog bilan yurishlar ===")
        nomzodlar = [("logreg", LogisticRegression(C=1.0, max_iter=1000)),
                     ("rf_d4", RandomForestClassifier(n_estimators=100, max_depth=4,
                                                      random_state=0)),
                     ("rf_d8", RandomForestClassifier(n_estimators=100, max_depth=8,
                                                      random_state=0)),
                     ("hgb", HistGradientBoostingClassifier(max_iter=100,
                                                            random_state=0))]
        for nom, model in nomzodlar:
            with tr.start_run("churn", nom) as r:
                autolog(r, model, Xtr, ytr, Xva, yva)
        try:
            with tr.start_run("churn", "rf_xato") as r:
                autolog(r, RandomForestClassifier(max_depth=-1), Xtr, ytr, Xva, yva)
        except Exception as x:
            print(f"  rf_xato: {type(x).__name__} -> holat FAILED")
        papka = tr.ildiz / "churn"
        print(f"  mlruns/churn ichida: {len(list(papka.iterdir()))} yurish papkasi")
        bitta = next(p for p in sorted(papka.iterdir())
                     if json.loads((p / "meta.json").read_text())["nom"] == "hgb")
        print(f"  hgb yurishi: {sorted(p.name for p in bitta.iterdir())}")
        print(f"  hgb params: {len(list((bitta / 'params').iterdir()))} ta "
              f"(get_params dan avtomatik), teglar: "
              f"{sorted(p.name for p in (bitta / 'tags').iterdir())}")

        print("\n=== 2. search_runs: filtr va tartib ===")
        for r in tr.search_runs("churn"):
            print(f"  {r['nom']:<7} {r['run_id']}  val_auc {r['metrics']['val_auc']:.4f}"
                  f"  {r['tags']['model_turi']}")
        rf = tr.search_runs("churn", lambda r: r["tags"]["model_turi"] ==
                            "RandomForestClassifier")
        print(f"  faqat RandomForest: {[(r['nom'], r['params']['max_depth']) for r in rf]}")

        print("\n=== 3. Eng yaxshi yurishni faqat yozuvdan qayta tiklash ===")
        eng = tr.search_runs("churn")[0]
        auc, sha_mos, bir_xil = qayta_tiklash(eng, Xtr, ytr, Xva, yva)
        print(f"  {eng['nom']}: yozuvda {eng['metrics']['val_auc']:.6f}, "
              f"qayta {auc:.6f}, teng: {auc == eng['metrics']['val_auc']}")
        print(f"  artefakt sha256 mos: {sha_mos}; saqlangan va qayta qurilgan "
              f"bashoratlar bir xil: {bir_xil}")

        print("\n=== 4. Ma'lumot o'zgargan bo'lsa - rad etish ===")
        Xtr2 = Xtr.copy()
        Xtr2[0, 0] += 1e-9
        try:
            qayta_tiklash(eng, Xtr2, ytr, Xva, yva)
        except ValueError as x:
            print(f"  ValueError: {x}")


if __name__ == "__main__":
    main()

=== 1. Autolog bilan yurishlar ===
  rf_xato: InvalidParameterError -> holat FAILED
  mlruns/churn ichida: 5 yurish papkasi
  hgb yurishi: ['artifacts', 'meta.json', 'metrics', 'params', 'tags']
  hgb params: 21 ta (get_params dan avtomatik), teglar: ['kod', 'malumot.oquv', 'malumot.val', 'model_turi', 'sha256.model.joblib', 'sklearn']

=== 2. search_runs: filtr va tartib ===
  hgb     8574a368  val_auc 0.7459  HistGradientBoostingClassifier
  rf_d8   94f45615  val_auc 0.7302  RandomForestClassifier
  logreg  1696e759  val_auc 0.7246  LogisticRegression
  rf_d4   e00299b5  val_auc 0.7163  RandomForestClassifier
  faqat RandomForest: [('rf_d8', 8), ('rf_d4', 4)]

=== 3. Eng yaxshi yurishni faqat yozuvdan qayta tiklash ===
  hgb: yozuvda 0.745891, qayta 0.745891, teng: True
  artefakt sha256 mos: True; saqlangan va qayta qurilgan bashoratlar bir xil: True

=== 4. Ma'lumot o'zgargan bo'lsa - rad etish ===
  ValueError: o'quv ma'lumoti xeshi mos emas - qayta tiklash mumkin emas

"""G'olib la'nati: ko'p yurishdan validatsiyada eng yaxshisi - optimistik."""

import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score

_rng = np.random.default_rng(2)
X = _rng.normal(size=(7000, 20))
# har belgi TENG va kuchsiz foydali -> nomzodlarning haqiqiy sifati deyarli bir xil
Y = (_rng.random(7000) < 1 / (1 + np.exp(-0.18 * X.sum(1)))).astype(int)
X_TEST, Y_TEST = X[2000:], Y[2000:]            # katta yopiq test (5000)


def nomzodlar(k, rng):
    """Tasodifiy qidiruv: C va 10 ta belgi."""
    return [{"C": float(10 ** rng.uniform(-3, 1)),
             "belgilar": sorted(rng.choice(20, size=10, replace=False).tolist())}
            for _ in range(k)]


def yurish(p, Xtr, ytr, Xva, yva):
    m = LogisticRegression(C=p["C"], max_iter=500).fit(Xtr[:, p["belgilar"]], ytr)
    val = roc_auc_score(yva, m.predict_proba(Xva[:, p["belgilar"]])[:, 1])
    test = roc_auc_score(Y_TEST, m.predict_proba(X_TEST[:, p["belgilar"]])[:, 1])
    return val, test


def main() -> None:
    K_MAX, TAKROR = 60, 12
    natija = {k: {"val": [], "test": []} for k in (1, 5, 20, 60)}
    birinchi = None
    for r in range(TAKROR):
        rng = np.random.default_rng(100 + r)
        idx = rng.permutation(2000)
        tr, va = idx[:1000], idx[1000:1300]           # validatsiya: 300 qator
        yurishlar = [yurish(p, X[tr], Y[tr], X[va], Y[va])
                     for p in nomzodlar(K_MAX, rng)]
        if birinchi is None:
            birinchi = yurishlar
        for k in natija:
            eng = max(yurishlar[:k], key=lambda v: v[0])    # validatsiya bo'yicha
            natija[k]["val"].append(eng[0])
            natija[k]["test"].append(eng[1])

    print("=== 1. Bitta qidiruv (60 yurish), validatsiya 300 qator ===")
    val = np.array([v for v, _ in birinchi])
    test = np.array([t for _, t in birinchi])
    i = int(val.argmax())
    print(f"  val AUC oralig'i:  {val.min():.4f} - {val.max():.4f}")
    print(f"  test AUC oralig'i: {test.min():.4f} - {test.max():.4f}")
    print(f"  eng yaxshi (val): {val[i]:.4f}; uning test AUC: {test[i]:.4f}")
    print(f"  test bo'yicha haqiqiy eng yaxshisi: {test.max():.4f}")
    print(f"  val va test korrelyatsiyasi (60 yurish): "
          f"{np.corrcoef(val, test)[0, 1]:.2f}")

    print(f"\n=== 2. {TAKROR} takror: yurishlar soni K va optimizm ===")
    print(f"  {'K':>3} {'val (eng yaxshi)':>16} {'test':>8} {'optimizm':>9} {'SE':>7}")
    for k, v in natija.items():
        o = np.array(v["val"]) - np.array(v["test"])
        print(f"  {k:>3} {np.mean(v['val']):>16.4f} {np.mean(v['test']):>8.4f} "
              f"{o.mean():>+9.4f} {o.std(ddof=1) / np.sqrt(len(o)):>7.4f}")

    print("\n=== 3. K=60 va K=1: juftlashgan farqlar (bir xil takrorlar) ===")
    for nom, kalit in [("val bahosi", "val"), ("test (haqiqiy)", "test")]:
        d = np.array(natija[60][kalit]) - np.array(natija[1][kalit])
        se = d.std(ddof=1) / np.sqrt(len(d))
        baho = "SEZILARLI" if abs(d.mean()) > 2 * se else "sezilarli emas"
        print(f"  {nom:<15} {d.mean():+.4f} (SE {se:.4f}) - {baho}")
    print("  hisobotga: K ni va tanlangan yurishning YOPIQ testdagi natijasini")


if __name__ == "__main__":
    main()

=== 1. Bitta qidiruv (60 yurish), validatsiya 300 qator ===
  val AUC oralig'i:  0.5888 - 0.6863
  test AUC oralig'i: 0.6235 - 0.6486
  eng yaxshi (val): 0.6863; uning test AUC: 0.6434
  test bo'yicha haqiqiy eng yaxshisi: 0.6486
  val va test korrelyatsiyasi (60 yurish): 0.31

=== 2. 12 takror: yurishlar soni K va optimizm ===
    K val (eng yaxshi)     test  optimizm      SE
    1           0.6443   0.6383   +0.0060  0.0062
    5           0.6646   0.6358   +0.0288  0.0061
   20           0.6746   0.6357   +0.0389  0.0052
   60           0.6836   0.6417   +0.0419  0.0049

=== 3. K=60 va K=1: juftlashgan farqlar (bir xil takrorlar) ===
  val bahosi      +0.0393 (SE 0.0059) - SEZILARLI
  test (haqiqiy)  +0.0034 (SE 0.0017) - sezilarli emas
  hisobotga: K ni va tanlangan yurishning YOPIQ testdagi natijasini

joblib.dump(model, "model_final_v3_haqiqiy.pkl")             # ⚠️ kontekst yo'q
with tr.start_run("churn", "gb_lr03") as r: autolog(r, model, ...)   # ✅ to'liq yozuv

if auc_yangi > auc_eski: ...                                 # ⚠️ val to'plamlari boshqa
assert a["malumot"] == b["malumot"]; farq, se = juft_farq(...)   # ✅

if auc > eng_yaxshi: log_run(...)                            # ⚠️ K yo'qoladi
with start_run(...): ...   # har yurish, FAILED ham           # ✅

r.log_metric("val_auc", yakuniy_auc)                         # ⚠️ egri chiziq yo'q
r.log_metric("val_auc", auc, qadam=davr)                     # ✅ har qadamda

shutil.copy("model.joblib", "artefaktlar/")                  # ⚠️ almashtirilsa - sezilmaydi
r.log_artifact("model.joblib", "model.joblib")  # + sha256    # ✅

print("Model AUC:", max(val_auclar))                         # ⚠️ g'olib la'nati
print("Model AUC (yopiq test):", test_auc, "K =", len(val_auclar))   # ✅

r.log_param("db_parol", os.environ["DB_PAROL"])              # ⚠️ kuzatuvchida hamma ko'radi
r.log_param("db_host", konfig.db_host)  # parol - faqat muhitda   # ✅

1.  # fayl nomlari bilan versiyalashning uch asosiy nuqsoni?
2.  # har yurishda qanday guruh ma'lumotlar yoziladi?
3.  # param va metrika farqi?
4.  # nega metrika qadamlar bilan yoziladi?
5.  # ikki yurishni solishtirish sharti?
6.  # ikki yurish farqining SE si uchun nima kerak?
7.  # muvaffaqiyatsiz yurishlar nega yoziladi?
8.  # g'olib la'nati nima?
9.  # K oshsa optimizm qanday o'zgaradi?
10. # hisobotga qaysi raqam yoziladi?
11. # autolog nima beradi?
12. # yozuvdan qayta tiklash nimani tekshiradi?

1.  joblib.dump(model, f"model_{auc:.3f}_final.pkl")

2.  if yangi["val_auc"] > eski["val_auc"]:
        print("yangi yaxshiroq")

3.  for p in tarmoq:
        auc = orgat(p)
        if auc > eng:
            eng = auc
            yozuv.append({"param": p, "auc": auc})

4.  r.log_param("learning_rate", 0.1)
    # ... max_depth, subsample va boshqalar unutildi

5.  print(f"Yakuniy model AUC: {max(r['val_auc'] for r in yurishlar):.3f}")