20.7-dars: Gradient tushish va optimizatorlar

QADAM:  w <- w - lr * dL/dw

lr JUDA KICHIK:
  har qadam arzimas, minimalgacha minglab davr
lr TO'G'RI:
  loss barqaror kamayadi
lr JUDA KATTA:
  minimumdan sakrab o'tadi, loss o'sadi yoki NaN

KVADRATIK FUNKSIYADA CHEGARA:
  L(w) = 0.5 * a * w^2   uchun
  lr < 2/a  bo'lsa yaqinlashadi
  lr > 2/a  bo'lsa uzoqlashadi

TARMOQDA: a - Gessian matritsasining eng katta xos qiymati
  uni hisoblash qimmat -> lr TAJRIBA bilan tanlanadi

TO'LIQ BATCH (batch = N):
  gradient ANIQ, har qadam qimmat
  tekis yo'l, lokal minimumda qolib ketishi mumkin

STOXASTIK (batch = 1):
  gradient juda shovqinli, har qadam arzon
  shovqin lokal minimumdan chiqishga yordam beradi

MINI-BATCH (batch = 32..512):
  amaliyotdagi standart
  matritsa amallaridan foydalanadi + foydali shovqin qoladi

GRADIENT SHOVQINI ~ 1/sqrt(batch)
  batch 4x katta -> shovqin 2x kichik

EMPIRIK QOIDA: batch ni 2x oshirsangiz, lr ni ham ~2x oshiring
               (yoki sqrt(2)x - manbalar kelishmaydi)

MUAMMO: uzun, tor "jarlik" da gradient tushish ZIG-ZAG qiladi
  tik devorlar bo'ylab tebranadi, jarlik bo'ylab sekin siljiydi

MOMENT (Polyak 1964):
  v <- beta * v + dL/dw
  w <- w - lr * v

  beta = 0.9  ->  o'rtacha ~10 qadam "eslanadi"

TA'SIRI:
  bir yo'nalishdagi gradientlar YIG'ILADI  -> tezlashadi
  tebranayotgan gradientlar BEKOR BO'LADI  -> tinchlanadi

NESTEROV varianti: gradientni "oldinga qarab" hisoblaydi
  amalda biroz yaxshiroq

MUAMMO: turli parametrlar turli masshtabda
  ba'zilarida gradient 1e-1, ba'zilarida 1e-5
  BITTA lr ikkalasiga ham mos kelmaydi

ADAGRAD:  s <- s + g^2;   w <- w - lr * g / (sqrt(s) + eps)
  kamchilik: s o'sib boradi -> qadam nolga tushadi

RMSPROP:  s <- rho*s + (1-rho)*g^2;   w <- w - lr*g/(sqrt(s)+eps)
  eksponensial o'rtacha -> s to'yinmaydi

ADAM (Kingma & Ba 2014) = MOMENT + RMSPROP + BIAS TUZATISH:
  m <- b1*m + (1-b1)*g          (birinchi moment)
  s <- b2*s + (1-b2)*g^2        (ikkinchi moment)
  m_hat = m / (1 - b1^t)        (bias tuzatish)
  s_hat = s / (1 - b2^t)
  w <- w - lr * m_hat / (sqrt(s_hat) + eps)

  sukut: b1=0.9, b2=0.999, eps=1e-8, lr=0.001

m ni NOL bilan boshlaymiz. Birinchi qadamda:
  m = 0.9*0 + 0.1*g = 0.1*g       <- haqiqiy gradientning 10% i!

Bu boshida qadamlarni sun'iy kichik qiladi.

TUZATISH: m_hat = m / (1 - 0.9^t)
  t=1:  m / 0.1   = g             <- to'g'ri
  t=10: m / 0.651
  t=100: m / 0.99997 ~ m          <- ta'siri yo'qoladi

Bias tuzatish faqat BIRINCHI o'nlab qadamda muhim,
lekin o'sha qadamlar butun o'rgatishning yo'nalishini belgilaydi

NIMA UCHUN: boshida katta qadam (tez harakat),
            oxirida kichik qadam (aniq sozlash)

STEP:       har K davrda lr ni gamma ga ko'paytirish
EXPONENTIAL: lr_t = lr_0 * gamma^t
COSINE:     lr_t = lr_min + 0.5*(lr_0-lr_min)*(1+cos(pi*t/T))
WARMUP:     birinchi bir necha davr lr ni 0 dan oshirish
            (transformerlarda MAJBURIY - 26-qism)
ONE-CYCLE:  warmup + cosine pasayish (juda samarali)

AMALIY: cosine + qisqa warmup - ko'p holatda eng yaxshi boshlanish

import numpy as np

# SGD + moment
v = np.zeros_like(w)
v = beta * v + g
w -= lr * v

# RMSProp
s = rho * s + (1 - rho) * g ** 2
w -= lr * g / (np.sqrt(s) + 1e-8)

# Adam
m = b1 * m + (1 - b1) * g
s = b2 * s + (1 - b2) * g ** 2
w -= lr * (m / (1 - b1 ** t)) / (np.sqrt(s / (1 - b2 ** t)) + 1e-8)

# cosine jadval
lr_t = lr_min + 0.5 * (lr_0 - lr_min) * (1 + np.cos(np.pi * t / T))

# mini-batch
for boshi in range(0, len(X), batch):
    idx = tartib[boshi:boshi + batch]

lr - eng muhim giperparametr
batch 32..512, shovqin ~ 1/sqrt(batch)
moment: inersiya, beta=0.9
Adam = moment + adaptiv + bias tuzatish
jadval: cosine + warmup

"""lr ning ta'siri: yaqinlashish, sekinlik, portlash (real numpy)."""

import numpy as np


def main() -> None:
    print("=== 1. Kvadratik funksiyada nazariy chegara ===")
    a = 2.0
    print(f"  L(w) = 0.5 * {a} * w^2,  chegara lr < 2/a = {2 / a}")
    print(f"  {'lr':>7} {'50 qadamdan keyin w':>22} {'holat':>12}")
    for lr in [0.05, 0.4, 0.9, 1.0, 1.05]:
        w = 1.0
        for _ in range(50):
            w = w - lr * a * w
            if not np.isfinite(w):
                break
        holat = "yaqinlashdi" if abs(w) < 1e-3 else (
            "sekin" if np.isfinite(w) and abs(w) < 1 else "uzoqlashdi")
        print(f"  {lr:>7.2f} {w:>22.6e} {holat:>12}")

    print("\n=== 2. Ikki o'lchovli 'jarlik' ===")
    A = np.array([20.0, 1.0])          # bir o'q 20x tik
    print(f"  L(w) = 0.5*(20*w1^2 + 1*w2^2)")
    print(f"  chegara: lr < 2/20 = 0.1")
    print(f"  {'lr':>7} {'qadamlar':>10} {'|w|':>12} {'izoh'}")
    for lr in [0.005, 0.05, 0.09, 0.11]:
        w = np.array([1.0, 1.0])
        qadam = 0
        # lr chegaradan katta bo'lsa w ATAYLAB cheksizlikka ketadi
        with np.errstate(over="ignore", invalid="ignore"):
            for qadam in range(1, 2001):
                w = w - lr * A * w
                if not np.isfinite(w).all() or np.linalg.norm(w) < 1e-4:
                    break
        with np.errstate(over="ignore", invalid="ignore"):
            norma = float(np.linalg.norm(w))
        izoh = ("yaqinlashdi" if norma < 1e-4
                else "portladi" if not np.isfinite(w).all() else "sekin")
        print(f"  {lr:>7.3f} {qadam:>10} {norma:>12.3e} {izoh}")
    print("  tik o'q lr ni CHEKLAYDI, yassi o'q sekin harakatlanadi")

    print("\n=== 3. Real loss da lr izlash ===")
    rng = np.random.default_rng(0)
    n, d = 500, 10
    X = rng.normal(0, 1, (n, d))
    w_haqiqiy = rng.normal(0, 1, d)
    y = X @ w_haqiqiy + rng.normal(0, 0.5, n)
    print(f"  {'lr':>8} {'100 qadamdan keyin loss':>26} {'holat':>12}")
    for lr in [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1.0]:
        w = np.zeros(d)
        with np.errstate(over="ignore", invalid="ignore"):
            for _ in range(100):
                g = 2 * X.T @ (X @ w - y) / n
                w = w - lr * g
                if not np.isfinite(w).all():
                    break
            loss = (float(np.mean((X @ w - y) ** 2))
                    if np.isfinite(w).all() else float("inf"))
        holat = ("portladi" if not np.isfinite(loss)
                 else "yaxshi" if loss < 0.3 else "sekin")
        print(f"  {lr:>8.0e} {loss:>26.6f} {holat:>12}")

    print("\n=== 4. lr izlash egri chizig'i ===")
    print("  kichik lr dan boshlab asta oshiramiz (LR range test):")
    print(f"  {'lr':>9} {'loss (10 qadam)':>18}")
    for lr in np.logspace(-4, 0.3, 9):
        w = np.zeros(d)
        for _ in range(10):
            g = 2 * X.T @ (X @ w - y) / n
            w = w - lr * g
        loss = (np.mean((X @ w - y) ** 2)
                if np.isfinite(w).all() else float("inf"))
        print(f"  {lr:>9.4f} {loss:>18.4f}")
    print("  eng past nuqtadan 3-10x kichigini tanlash odat")

    print("\n=== 5. Qadamlar traektoriyasi ===")
    print(f"  jarlikda lr = 0.09 bilan birinchi qadamlar:")
    w = np.array([1.0, 1.0])
    print(f"  {'qadam':>6} {'w1':>12} {'w2':>12}")
    for i in range(1, 9):
        w = w - 0.09 * A * w
        print(f"  {i:>6} {w[0]:>12.6f} {w[1]:>12.6f}")
    print("  ⭐ w1 tebranadi, w2 sekin siljiydi - moment shuni tuzatadi")


if __name__ == "__main__":
    main()

=== 1. Kvadratik funksiyada nazariy chegara ===
  L(w) = 0.5 * 2.0 * w^2,  chegara lr < 2/a = 1.0
       lr    50 qadamdan keyin w        holat
     0.05           5.153775e-03        sekin
     0.40           1.125900e-35  yaqinlashdi
     0.90           1.427248e-05  yaqinlashdi
     1.00           1.000000e+00   uzoqlashdi
     1.05           1.173909e+02   uzoqlashdi

=== 2. Ikki o'lchovli 'jarlik' ===
  L(w) = 0.5*(20*w1^2 + 1*w2^2)
  chegara: lr < 2/20 = 0.1
       lr   qadamlar          |w| izoh
    0.005       1838    9.973e-05 yaqinlashdi
    0.050        180    9.778e-05 yaqinlashdi
    0.090         98    9.684e-05 yaqinlashdi
    0.110       2000          inf sekin
  tik o'q lr ni CHEKLAYDI, yassi o'q sekin harakatlanadi

=== 3. Real loss da lr izlash ===
        lr    100 qadamdan keyin loss        holat
     1e-04                   8.280343        sekin
     1e-03                   5.724921        sekin
     1e-02                   0.381762        sekin
     1e-01                   0.254561       yaxshi
     5e-01                   0.254561       yaxshi
     1e+00 27495031550720895575691757889681096704.000000        sekin

=== 4. lr izlash egri chizig'i ===
  kichik lr dan boshlab asta oshiramiz (LR range test):
         lr    loss (10 qadam)
     0.0001             8.5945
     0.0003             8.5078
     0.0012             8.2156
     0.0041             7.2825
     0.0141             4.8073
     0.0487             1.2208
     0.1679             0.2569
     0.5788             0.2546
     1.9953 1865905289412.8137
  eng past nuqtadan 3-10x kichigini tanlash odat

=== 5. Qadamlar traektoriyasi ===
  jarlikda lr = 0.09 bilan birinchi qadamlar:
   qadam           w1           w2
       1    -0.800000     0.910000
       2     0.640000     0.828100
       3    -0.512000     0.753571
       4     0.409600     0.685750
       5    -0.327680     0.624032
       6     0.262144     0.567869
       7    -0.209715     0.516761
       8     0.167772     0.470253
  ⭐ w1 tebranadi, w2 sekin siljiydi - moment shuni tuzatadi

"""Mini-batch gradienti aniq gradientdan qancha farq qiladi."""

import numpy as np


def main() -> None:
    rng = np.random.default_rng(0)
    n, d = 4000, 8
    X = rng.normal(0, 1, (n, d))
    w_haqiqiy = rng.normal(0, 1, d)
    y = X @ w_haqiqiy + rng.normal(0, 1.0, n)
    w = rng.normal(0, 0.5, d)

    def gradient(idx):
        Xb, yb = X[idx], y[idx]
        return 2 * Xb.T @ (Xb @ w - yb) / len(idx)

    aniq = gradient(np.arange(n))

    print("=== 1. Batch hajmi va gradient shovqini ===")
    print(f"  {'batch':>7} {'o_rtacha burchak':>18} "
          f"{'nisbiy xato':>13} {'1/sqrt(b)':>11}")
    for b in [1, 4, 16, 64, 256, 1024]:
        burchaklar, xatolar = [], []
        for _ in range(60):
            idx = rng.choice(n, b, replace=False)
            g = gradient(idx)
            kos = (g @ aniq
                   / (np.linalg.norm(g) * np.linalg.norm(aniq)))
            burchaklar.append(np.degrees(np.arccos(np.clip(kos, -1, 1))))
            xatolar.append(np.linalg.norm(g - aniq)
                           / np.linalg.norm(aniq))
        print(f"  {b:>7} {np.mean(burchaklar):>17.2f}° "
              f"{np.mean(xatolar):>13.4f} {1 / np.sqrt(b):>11.4f}")
    print("  nisbiy xato ~ 1/sqrt(batch) ga mutanosib")

    print("\n=== 2. Bir davrda necha qadam ===")
    print(f"  {'batch':>7} {'qadamlar/davr':>15} "
          f"{'gradient hisoblari':>20}")
    for b in [1, 32, 256, 4000]:
        print(f"  {b:>7} {int(np.ceil(n / b)):>15} {n:>20}")
    print("  hisob hajmi BIR XIL, qadamlar soni boshqa")

    print("\n=== 3. Bir xil davrda natija ===")
    rng2 = np.random.default_rng(1)
    print(f"  {'batch':>7} {'lr':>8} {'5 davrdan keyin loss':>22}")
    for b, lr in [(1, 0.002), (32, 0.01), (256, 0.05), (4000, 0.05)]:
        ww = np.zeros(d)
        for _ in range(5):
            tartib = rng2.permutation(n)
            for boshi in range(0, n, b):
                idx = tartib[boshi:boshi + b]
                g = 2 * X[idx].T @ (X[idx] @ ww - y[idx]) / len(idx)
                ww = ww - lr * g
        print(f"  {b:>7} {lr:>8.3f} "
              f"{np.mean((X @ ww - y) ** 2):>22.6f}")
    print("  kichik batch ko'p qadam tashlaydi - tezroq yaqinlashadi")

    print("\n=== 4. Batch va lr birga o'zgaradi ===")
    print(f"  {'batch':>7} {'lr = 0.01':>12} {'lr ~ batch':>12}")
    for b in [32, 64, 128, 256]:
        natijalar = []
        for lr in [0.01, 0.01 * b / 32]:
            ww = np.zeros(d)
            r = np.random.default_rng(2)
            for _ in range(5):
                tartib = r.permutation(n)
                for boshi in range(0, n, b):
                    idx = tartib[boshi:boshi + b]
                    g = (2 * X[idx].T @ (X[idx] @ ww - y[idx])
                         / len(idx))
                    ww = ww - lr * g
            natijalar.append(np.mean((X @ ww - y) ** 2))
        print(f"  {b:>7} {natijalar[0]:>12.6f} {natijalar[1]:>12.6f}")
    print("  batch oshganda lr ni ham oshirish kerak")

    print("\n=== 5. Shovqin foydali bo'lgan holat ===")
    def egri(w):
        return float(w ** 2 + 3 * np.sin(4 * w) ** 2)

    def egri_grad(w):
        return float(2 * w + 24 * np.sin(4 * w) * np.cos(4 * w))

    print("  L(w) = w^2 + 3*sin(4w)^2 - ko'p lokal minimum")
    print(f"  {'shovqin':>9} {'yakuniy w':>12} {'L(w)':>10}")
    for shovqin in [0.0, 0.5, 2.0, 6.0]:
        r = np.random.default_rng(3)
        w = 2.5
        for _ in range(400):
            g = egri_grad(w) + shovqin * r.normal()
            w = w - 0.01 * g
        print(f"  {shovqin:>9.1f} {w:>12.4f} {egri(w):>10.4f}")
    print("  ⭐ O'rtacha shovqin lokal minimumdan chiqishga yordam beradi")


if __name__ == "__main__":
    main()

=== 1. Batch hajmi va gradient shovqini ===
    batch   o_rtacha burchak   nisbiy xato   1/sqrt(b)
        1             69.29°        3.1063      1.0000
        4             55.07°        1.4687      0.5000
       16             33.41°        0.7641      0.2500
       64             18.91°        0.3885      0.1250
      256              9.40°        0.1890      0.0625
     1024              4.10°        0.0813      0.0312
  nisbiy xato ~ 1/sqrt(batch) ga mutanosib

=== 2. Bir davrda necha qadam ===
    batch   qadamlar/davr   gradient hisoblari
        1            4000                 4000
       32             125                 4000
      256              16                 4000
     4000               1                 4000
  hisob hajmi BIR XIL, qadamlar soni boshqa

=== 3. Bir xil davrda natija ===
    batch       lr   5 davrdan keyin loss
        1    0.002               1.047664
       32    0.010               1.035608
      256    0.050               1.035195
     4000    0.050               2.560870
  kichik batch ko'p qadam tashlaydi - tezroq yaqinlashadi

=== 4. Batch va lr birga o'zgaradi ===
    batch    lr = 0.01   lr ~ batch
       32     1.035626     1.035626
       64     1.035226     1.035995
      128     1.043110     1.037284
      256     1.208736     1.036383
  batch oshganda lr ni ham oshirish kerak

=== 5. Shovqin foydali bo'lgan holat ===
  L(w) = w^2 + 3*sin(4w)^2 - ko'p lokal minimum
    shovqin    yakuniy w       L(w)
        0.0       2.3069     5.4369
        0.5       2.3092     5.4372
        2.0       2.3160     5.4407
        6.0       2.3319     5.4660
  ⭐ O'rtacha shovqin lokal minimumdan chiqishga yordam beradi

"""SGD, moment, RMSProp, Adam bitta vazifada (real numpy)."""

import numpy as np


def optimizatorlar(nom, d):
    """Holat lug'atini qaytaradi."""
    return {"v": np.zeros(d), "s": np.zeros(d), "m": np.zeros(d),
            "t": 0}


def qadam(nom, w, g, h, lr):
    h["t"] += 1
    if nom == "SGD":
        return w - lr * g
    if nom == "Moment":
        h["v"] = 0.9 * h["v"] + g
        return w - lr * h["v"]
    if nom == "RMSProp":
        h["s"] = 0.9 * h["s"] + 0.1 * g ** 2
        return w - lr * g / (np.sqrt(h["s"]) + 1e-8)
    if nom == "Adam":
        h["m"] = 0.9 * h["m"] + 0.1 * g
        h["s"] = 0.999 * h["s"] + 0.001 * g ** 2
        m_hat = h["m"] / (1 - 0.9 ** h["t"])
        s_hat = h["s"] / (1 - 0.999 ** h["t"])
        return w - lr * m_hat / (np.sqrt(s_hat) + 1e-8)
    raise ValueError(nom)


def main() -> None:
    print("=== 1. 'Jarlik' funksiyasida ===")
    A = np.array([20.0, 1.0])
    print("  L(w) = 0.5*(20*w1^2 + w2^2), boshlanish (1, 1)")
    print(f"  {'optimizator':<12} {'lr':>7} {'200 qadamdan keyin |w|':>25}")
    for nom, lr in [("SGD", 0.05), ("Moment", 0.01),
                    ("RMSProp", 0.05), ("Adam", 0.05)]:
        w = np.array([1.0, 1.0])
        h = optimizatorlar(nom, 2)
        for _ in range(200):
            w = qadam(nom, w, A * w, h, lr)
        print(f"  {nom:<12} {lr:>7.3f} {np.linalg.norm(w):>25.4e}")

    print("\n=== 2. Har parametr uchun masshtab farqli ===")
    masshtab = np.array([1e-3, 1e-1, 1.0, 1e1])
    print(f"  gradient masshtablari: {masshtab}")
    print(f"  {'optimizator':<12} {'300 qadamdan keyin har |w|':>32}")
    for nom in ["SGD", "Moment", "RMSProp", "Adam"]:
        w = np.ones(4)
        h = optimizatorlar(nom, 4)
        for _ in range(300):
            w = qadam(nom, w, masshtab * w, h, 0.01)
        print(f"  {nom:<12} {str(np.abs(w).round(6)):>32}")
    print("  adaptiv usullar hamma o'qni BIR XIL tezlikda olib boradi")

    print("\n=== 3. Bias tuzatishning ta'siri ===")
    g = np.array([1.0])
    print(f"  doimiy gradient g = 1.0")
    print(f"  {'qadam':>6} {'m':>10} {'m_hat':>10} {'nisbat':>9}")
    m = np.zeros(1)
    for t in range(1, 9):
        m = 0.9 * m + 0.1 * g
        m_hat = m / (1 - 0.9 ** t)
        print(f"  {t:>6} {m[0]:>10.6f} {m_hat[0]:>10.6f} "
              f"{m_hat[0] / m[0]:>9.3f}")
    print("  tuzatishsiz birinchi qadam 10 barobar kichik bo'lardi")

    print("\n=== 4. Real regressiya vazifasida ===")
    rng = np.random.default_rng(0)
    n, d = 2000, 20
    X = rng.normal(0, 1, (n, d)) * np.logspace(-1, 1, d)
    w_haqiqiy = rng.normal(0, 1, d)
    y = X @ w_haqiqiy + rng.normal(0, 0.5, n)
    print("  belgilar masshtabi 0.1 dan 10 gacha (ataylab)")
    print(f"  {'optimizator':<12} {'lr':>7}", end="")
    for davr in [5, 20, 100]:
        print(f" {f'{davr} davr':>12}", end="")
    print()
    for nom, lr in [("SGD", 0.002), ("Moment", 0.0005),
                    ("RMSProp", 0.05), ("Adam", 0.05)]:
        w = np.zeros(d)
        h = optimizatorlar(nom, d)
        chiqish = []
        for davr in range(1, 101):
            g = 2 * X.T @ (X @ w - y) / n
            w = qadam(nom, w, g, h, lr)
            if davr in (5, 20, 100):
                chiqish.append(float(np.mean((X @ w - y) ** 2)))
        print(f"  {nom:<12} {lr:>7.4f}", end="")
        for c in chiqish:
            print(f" {c:>12.4f}", end="")
        print()

    print("\n=== 5. lr ga sezgirlik ===")
    print(f"  {'optimizator':<12}", end="")
    lrlar = [1e-4, 1e-3, 1e-2, 1e-1]
    for lr in lrlar:
        print(f" {lr:>11.0e}", end="")
    print()
    for nom in ["SGD", "Moment", "RMSProp", "Adam"]:
        print(f"  {nom:<12}", end="")
        for lr in lrlar:
            w = np.zeros(d)
            h = optimizatorlar(nom, d)
            for _ in range(50):
                g = 2 * X.T @ (X @ w - y) / n
                w = qadam(nom, w, g, h, lr)
                if not np.isfinite(w).all():
                    break
            loss = (float(np.mean((X @ w - y) ** 2))
                    if np.isfinite(w).all() else float("inf"))
            print(f" {loss:>11.3g}", end="")
        print()
    print("  ⭐ Har optimizatorning O'Z lr oralig'i bor")


if __name__ == "__main__":
    main()

=== 1. 'Jarlik' funksiyasida ===
  L(w) = 0.5*(20*w1^2 + w2^2), boshlanish (1, 1)
  optimizator       lr    200 qadamdan keyin |w|
  SGD            0.050                3.5053e-05
  Moment         0.010                1.4095e-05
  RMSProp        0.050                3.4882e-02
  Adam           0.050                4.0236e-05

=== 2. Har parametr uchun masshtab farqli ===
  gradient masshtablari: [1.e-03 1.e-01 1.e+00 1.e+01]
  optimizator        300 qadamdan keyin har |w|
  SGD          [0.997004 0.740707 0.049041 0.      ]
  Moment       [0.971294 0.03933  0.       0.      ]
  RMSProp      [0.004959 0.004961 0.00496  0.00496 ]
  Adam         [0.000183 0.000183 0.000183 0.000183]
  adaptiv usullar hamma o'qni BIR XIL tezlikda olib boradi

=== 3. Bias tuzatishning ta'siri ===
  doimiy gradient g = 1.0
   qadam          m      m_hat    nisbat
       1   0.100000   1.000000    10.000
       2   0.190000   1.000000     5.263
       3   0.271000   1.000000     3.690
       4   0.343900   1.000000     2.908
       5   0.409510   1.000000     2.442
       6   0.468559   1.000000     2.134
       7   0.521703   1.000000     1.917
       8   0.569533   1.000000     1.756
  tuzatishsiz birinchi qadam 10 barobar kichik bo'lardi

=== 4. Real regressiya vazifasida ===
  belgilar masshtabi 0.1 dan 10 gacha (ataylab)
  optimizator       lr       5 davr      20 davr     100 davr
  SGD           0.0020      52.4173      11.0975       2.4965
  Moment        0.0005      65.1189      42.4313       1.4509
  RMSProp       0.0500     184.1141      20.7563       0.3278
  Adam          0.0500     298.8404      51.4681       0.2554

=== 5. lr ga sezgirlik ===
  optimizator        1e-04       1e-03       1e-02       1e-01
  SGD                  124        8.55    3.01e+05   8.59e+131
  Moment               9.9        1.94        2.24   6.89e+129
  RMSProp              446         412         159       0.245
  Adam                 447         417         191        1.77
  ⭐ Har optimizatorning O'Z lr oralig'i bor

"""Learning rate schedule va sklearn bilan tekshiruv."""

import warnings

import numpy as np
from sklearn.datasets import make_classification
from sklearn.exceptions import ConvergenceWarning
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler


def jadval(nom, t, T, lr0=0.1, lr_min=0.001):
    if nom == "doimiy":
        return lr0
    if nom == "step":
        return lr0 * (0.1 ** (t // (T // 3)))
    if nom == "exponential":
        return lr0 * (0.01 ** (t / T))
    if nom == "cosine":
        return lr_min + 0.5 * (lr0 - lr_min) * (1 + np.cos(np.pi * t / T))
    if nom == "warmup+cosine":
        w = max(1, T // 10)
        if t < w:
            return lr0 * (t + 1) / w
        u = (t - w) / max(1, T - w)
        return lr_min + 0.5 * (lr0 - lr_min) * (1 + np.cos(np.pi * u))
    raise ValueError(nom)


def main() -> None:
    T = 40
    nomlar = ["doimiy", "step", "exponential", "cosine",
              "warmup+cosine"]

    print("=== 1. Jadvallar shakli ===")
    print(f"  {'davr':>6}", end="")
    for nom in nomlar:
        print(f" {nom:>14}", end="")
    print()
    for t in [0, 3, 10, 20, 30, 39]:
        print(f"  {t:>6}", end="")
        for nom in nomlar:
            print(f" {jadval(nom, t, T):>14.5f}", end="")
        print()

    print("\n=== 2. O'rtacha lr ===")
    print(f"  {'jadval':<16} {'o_rtacha lr':>13} {'oxirgi lr':>11}")
    for nom in nomlar:
        qiymatlar = [jadval(nom, t, T) for t in range(T)]
        print(f"  {nom:<16} {np.mean(qiymatlar):>13.5f} "
              f"{qiymatlar[-1]:>11.5f}")

    print("\n=== 3. Regressiyada jadval ta'siri ===")
    rng = np.random.default_rng(0)
    n, d = 1500, 15
    X = rng.normal(0, 1, (n, d)) * np.logspace(-0.5, 0.5, d)
    w_haq = rng.normal(0, 1, d)
    y = X @ w_haq + rng.normal(0, 0.4, n)
    print(f"  {'jadval':<16} {'yakuniy loss':>14} "
          f"{'eng yaxshi loss':>17}")
    for nom in nomlar:
        w = np.zeros(d)
        eng = float("inf")
        for t in range(T * 8):
            lr = jadval(nom, t, T * 8, lr0=0.05)
            g = 2 * X.T @ (X @ w - y) / n
            w = w - lr * g
            eng = min(eng, float(np.mean((X @ w - y) ** 2)))
        print(f"  {nom:<16} {np.mean((X @ w - y) ** 2):>14.6f} "
              f"{eng:>17.6f}")

    print("\n=== 4. sklearn optimizatorlari ===")
    Xc, yc = make_classification(n_samples=3000, n_features=20,
                                 n_informative=10, n_redundant=4,
                                 n_classes=3, flip_y=0.1,
                                 class_sep=0.9, random_state=0)
    Xtr, Xte, ytr, yte = train_test_split(Xc, yc, test_size=0.3,
                                          random_state=0, stratify=yc)
    sc = StandardScaler().fit(Xtr)
    Xtr, Xte = sc.transform(Xtr), sc.transform(Xte)
    print(f"  {'solver':<8} {'lr':>8} {'davrlar':>9} {'loss':>9} "
          f"{'test':>8}")
    for solver, lr in [("sgd", 0.001), ("sgd", 0.01), ("sgd", 0.1),
                       ("adam", 0.001), ("adam", 0.01), ("adam", 0.1)]:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", ConvergenceWarning)
            m = MLPClassifier(hidden_layer_sizes=(64, 32),
                              solver=solver, learning_rate_init=lr,
                              max_iter=150, random_state=0,
                              early_stopping=False).fit(Xtr, ytr)
        print(f"  {solver:<8} {lr:>8.3f} {m.n_iter_:>9} "
              f"{m.loss_:>9.4f} "
              f"{accuracy_score(yte, m.predict(Xte)):>8.4f}")
    print("  ⭐ Optimizatorni BIR XIL lr bilan taqqoslash noto'g'ri")

    print("\n=== 5. sklearn ning o'z jadvallari ===")
    print(f"  {'learning_rate':<12} {'davrlar':>9} {'loss':>9} "
          f"{'test':>8}")
    for rejim in ["constant", "invscaling", "adaptive"]:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", ConvergenceWarning)
            m = MLPClassifier(hidden_layer_sizes=(64, 32), solver="sgd",
                              learning_rate=rejim,
                              learning_rate_init=0.05, max_iter=150,
                              random_state=0,
                              early_stopping=False).fit(Xtr, ytr)
        print(f"  {rejim:<12} {m.n_iter_:>9} {m.loss_:>9.4f} "
              f"{accuracy_score(yte, m.predict(Xte)):>8.4f}")


if __name__ == "__main__":
    main()

=== 1. Jadvallar shakli ===
    davr         doimiy           step    exponential         cosine  warmup+cosine
       0        0.10000        0.10000        0.10000        0.10000        0.02500
       3        0.10000        0.10000        0.07079        0.09863        0.10000
      10        0.10000        0.10000        0.03162        0.08550        0.09337
      20        0.10000        0.01000        0.01000        0.05050        0.05910
      30        0.10000        0.00100        0.00316        0.01550        0.01868
      39        0.10000        0.00010        0.00112        0.00115        0.00119

=== 2. O'rtacha lr ===
  jadval             o_rtacha lr   oxirgi lr
  doimiy                 0.10000     0.10000
  step                   0.03608     0.00010
  exponential            0.02276     0.00112
  cosine                 0.05174     0.00115
  warmup+cosine          0.05294     0.00119

=== 3. Regressiyada jadval ta'siri ===
  jadval             yakuniy loss   eng yaxshi loss
  doimiy                 0.149416          0.149416
  step                   0.157593          0.157593
  exponential            0.197353          0.197353
  cosine                 0.151485          0.151485
  warmup+cosine          0.151475          0.151475

=== 4. sklearn optimizatorlari ===
  solver         lr   davrlar      loss     test
  sgd         0.001       150    0.5118   0.7644
  sgd         0.010       150    0.1671   0.7933
  sgd         0.100        98    0.0023   0.7989
  adam        0.001       150    0.1194   0.8089
  adam        0.010        87    0.0016   0.7944
  adam        0.100        53    0.2637   0.8089
  ⭐ Optimizatorni BIR XIL lr bilan taqqoslash noto'g'ri

=== 5. sklearn ning o'z jadvallari ===
  learning_rate   davrlar      loss     test
  constant           150    0.0037   0.7856
  invscaling         150    0.6469   0.7289
  adaptive           150    0.0037   0.7856

# SGD(lr=0.001) va Adam(lr=0.001)              # ⚠️
# har biriga o'z lr izlanadi                   # ✅

loss.backward(); opt.step()                    # ⚠️ torch da yig'iladi
opt.zero_grad(); loss.backward(); opt.step()   # ✅

batch = 512   # lr o'zgarmadi                   # ⚠️
lr = lr * (512 / 64)                            # ✅

Adam(params, weight_decay=0.01)                # ⚠️ L2 adaptiv bilan buziladi
AdamW(params, weight_decay=0.01)               # ✅

# sukut qiymat bilan qoldirish                 # ⚠️
# LR range test bilan izlash                   # ✅

w -= lr * g / np.sqrt(s)                       # ⚠️ 0 ga bo'lish
w -= lr * g / (np.sqrt(s) + 1e-8)              # ✅

# cosine har doim yaxshi deb o'ylash           # ⚠️
# validatsiya balli bilan tekshirish           # ✅

1.  # gradient tushish qadami?
2.  # kvadratikda lr chegarasi?
3.  # shovqin batch bilan qanday bog'liq?
4.  # moment nima qiladi?
5.  # beta = 0.9 necha qadam eslaydi?
6.  # RMSProp nimani hal qiladi?
7.  # Adam necha g'oyadan iborat?
8.  # bias tuzatish nima uchun?
9.  # Adam sukut lr?
10. # cosine jadval formulasi?
11. # warmup nima uchun?
12. # AdamW farqi?

1.  # SGD(lr=0.001) va Adam(lr=0.001) ni taqqoslash

2.  loss.backward(); opt.step()

3.  batch = 512   # lr o'zgarmadi

4.  Adam(params, weight_decay=0.01)

5.  w -= lr * g / np.sqrt(s)