21.4-dars: torch.optim va jadvallar

opt = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-2)

BIR QADAM:
  opt.zero_grad()      # .grad larni tozalash (set_to_none=True sukut)
  loss.backward()      # .grad larni hisoblash
  opt.step()           # parametrlarni yangilash

opt.param_groups   -> ro'yxat, har biri: {"params": [...], "lr": ...}
opt.state          -> har parametr uchun ichki holat
                      Adam: exp_avg (m), exp_avg_sq (s), step

KENG TARQALGANLAR:
  SGD(lr, momentum=0.9, nesterov=True)   CNN larda klassik
  AdamW(lr, weight_decay)                standart tanlov
  Adam                                   weight_decay siz ishlatilsa
  RMSprop, Adagrad                       maxsus holatlar

opt = torch.optim.AdamW([
    {"params": model.asos.parameters(), "lr": 1e-4},
    {"params": model.bosh.parameters(), "lr": 1e-3},
], weight_decay=1e-2)

GURUHDA BERILMAGAN parametr umumiy qiymatni oladi

QAYERDA ISHLATILADI:
  transfer learning: asos kichik lr, yangi bosh katta lr
  weight_decay ni bias va norm dan ajratish
  embedding uchun alohida lr
  qatlamma-qatlam kamayuvchi lr (layer-wise decay)

GURUHNI O'ZGARTIRISH:
  for g in opt.param_groups:
      g["lr"] = g["lr"] * 0.5

MUAMMO: weight_decay HAMMA parametrni nolga tortadi
  bias      -> siljishni cheklash ma'nosiz
  LayerNorm/BatchNorm og'irligi -> masshtabni buzadi
  embedding -> ba'zan kerak, ba'zan yo'q

AMALIY QOIDA (transformerlarda standart):
  2D og'irliklar (Linear.weight)   -> decay BOR
  1D parametrlar (bias, norm)       -> decay YO'Q

  decay, decaysiz = [], []
  for nom, p in model.named_parameters():
      if not p.requires_grad: continue
      (decaysiz if p.ndim < 2 else decay).append(p)
  opt = AdamW([{"params": decay, "weight_decay": 0.01},
               {"params": decaysiz, "weight_decay": 0.0}], lr=...)

sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=davrlar)

ASOSIYLARI:
  StepLR(step_size, gamma)         har N davrda ko'paytirish
  ExponentialLR(gamma)             har davr gamma ga
  CosineAnnealingLR(T_max)         kosinus bo'yicha pasayish
  OneCycleLR(max_lr, total_steps)  warmup + pasayish (HAR BATCH)
  LinearLR(start_factor, total_iters)  chiziqli warmup
  SequentialLR([warmup, asosiy], milestones=[k])  birlashtirish
  ReduceLROnPlateau(mode, patience)    validatsiyaga qarab

QADAMLASH TARTIBI:
  opt.step()  AVVAL
  sched.step() KEYIN

QACHON QADAMLASH:
  davr jadvallari (StepLR, Cosine T_max=davrlar) -> DAVR oxirida
  qadam jadvallari (OneCycleLR, T_max=qadamlar)  -> har BATCH
  ReduceLROnPlateau -> sched.step(val_loss)

NORMA BO'YICHA (eng keng tarqalgan):
  loss.backward()
  torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
  opt.step()

  barcha gradientlar birgalikda normalanadi
  yo'nalish SAQLANADI, uzunlik cheklanadi
  qaytaradi: clipping OLDIDAGI norma (kuzatish uchun foydali)

QIYMAT BO'YICHA:
  clip_grad_value_(params, clip_value=0.5)
  har elementni alohida qisadi - yo'nalish O'ZGARADI

QAYERDA ZARUR:
  RNN/LSTM, transformerlar, katta lr, beqaror loss

TARTIB: backward -> clip -> step

CHECKPOINT:
  torch.save({
      "model": model.state_dict(),
      "opt": opt.state_dict(),
      "sched": sched.state_dict(),
      "davr": davr,
  }, yol)

TIKLASH:
  paket = torch.load(yol, weights_only=True)
  model.load_state_dict(paket["model"])
  opt.load_state_dict(paket["opt"])
  sched.load_state_dict(paket["sched"])
  boshlanish = paket["davr"] + 1

NIMA UCHUN OPTIMIZATOR HOLATI:
  Adam ning m va s si saqlanmasa, davom ettirishda
  optimizator "noldan" boshlaydi -> loss sakraydi

import torch

def guruhlar(model, wd=0.01):
    decay, decaysiz = [], []
    for p in model.parameters():
        if p.requires_grad:
            (decaysiz if p.ndim < 2 else decay).append(p)
    return [{"params": decay, "weight_decay": wd},
            {"params": decaysiz, "weight_decay": 0.0}]

opt = torch.optim.AdamW(guruhlar(model), lr=3e-3)
warmup = torch.optim.lr_scheduler.LinearLR(opt, start_factor=0.1,
                                           total_iters=5)
asosiy = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=45)
sched = torch.optim.lr_scheduler.SequentialLR(
    opt, [warmup, asosiy], milestones=[5])

for davr in range(50):
    for xb, yb in dl:
        opt.zero_grad()
        loss = kriteriy(model(xb), yb)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()
    sched.step()                     # DAVR oxirida

zero_grad -> backward -> clip -> step -> (sched.step)
parametr guruhlari: turli lr / wd
p.ndim < 2 -> decay yo'q
jadval birligi = step() chaqiruv birligi
checkpoint: model + opt + sched

"""param_groups va state: optimizator nimani saqlaydi (real torch)."""

import torch
import torch.nn as nn


def main() -> None:
    torch.manual_seed(0)
    model = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 2))
    opt = torch.optim.AdamW(model.parameters(), lr=1e-2,
                            weight_decay=1e-2)

    print("=== 1. param_groups ===")
    print(f"  guruhlar soni: {len(opt.param_groups)}")
    g = opt.param_groups[0]
    kalitlar = sorted(k for k in g if k != "params")
    print(f"  guruh kalitlari: {kalitlar[:8]}")
    print(f"  lr = {g['lr']}, weight_decay = {g['weight_decay']}, "
          f"betas = {g['betas']}")
    print(f"  guruhdagi parametrlar: {len(g['params'])}")

    print("\n=== 2. Holat boshida bo'sh ===")
    print(f"  opt.state uzunligi: {len(opt.state)}")

    X = torch.randn(16, 4)
    y = torch.randint(0, 2, (16,))
    kriteriy = nn.CrossEntropyLoss()
    for _ in range(3):
        opt.zero_grad()
        kriteriy(model(X), y).backward()
        opt.step()

    print("\n=== 3. Uch qadamdan keyin holat ===")
    print(f"  opt.state uzunligi: {len(opt.state)}")
    birinchi = model[0].weight
    holat = opt.state[birinchi]
    print(f"  {'kalit':<12} {'shakl':>10} {'norma':>12}")
    for k, v in holat.items():
        if torch.is_tensor(v) and v.ndim > 0:
            print(f"  {k:<12} {str(tuple(v.shape)):>10} "
                  f"{v.norm().item():>12.6f}")
        else:
            print(f"  {k:<12} {'skalyar':>10} {float(v):>12.1f}")
    print("  exp_avg = m, exp_avg_sq = s (20.7-dars)")

    print("\n=== 4. zero_grad set_to_none ===")
    opt.zero_grad()
    print(f"  zero_grad() dan keyin grad: "
          f"{model[0].weight.grad}")
    opt.zero_grad(set_to_none=False)
    kriteriy(model(X), y).backward()
    opt.zero_grad(set_to_none=False)
    print(f"  set_to_none=False bilan grad normasi: "
          f"{model[0].weight.grad.norm().item():.1f}")
    print("  None - xotira tejaladi va 'grad yo'q' aniq ko'rinadi")

    print("\n=== 5. Qo'lda Adam bilan solishtirish ===")
    torch.manual_seed(1)
    w = torch.randn(5, requires_grad=True)
    w_qolda = w.detach().clone()
    opt2 = torch.optim.Adam([w], lr=0.1)
    m = torch.zeros(5)
    s = torch.zeros(5)
    for t in range(1, 6):
        opt2.zero_grad()
        (w ** 2).sum().backward()
        opt2.step()
        g2 = 2 * w_qolda
        m = 0.9 * m + 0.1 * g2
        s = 0.999 * s + 0.001 * g2 ** 2
        m_hat = m / (1 - 0.9 ** t)
        s_hat = s / (1 - 0.999 ** t)
        w_qolda = w_qolda - 0.1 * m_hat / (s_hat.sqrt() + 1e-8)
    print(f"  torch.optim.Adam: {w.detach().numpy().round(6)}")
    print(f"  qo'lda 20.7-bob:    {w_qolda.numpy().round(6)}")
    print(f"  bir xilmi: {torch.allclose(w.detach(), w_qolda, atol=1e-6)}")
    print("  ⭐ torch.optim - bizning formulalar, boshqa hech narsa")


if __name__ == "__main__":
    main()

=== 1. param_groups ===
  guruhlar soni: 1
  guruh kalitlari: ['amsgrad', 'betas', 'capturable', 'decoupled_weight_decay', 'differentiable', 'eps', 'foreach', 'fused']
  lr = 0.01, weight_decay = 0.01, betas = (0.9, 0.999)
  guruhdagi parametrlar: 4

=== 2. Holat boshida bo'sh ===
  opt.state uzunligi: 0

=== 3. Uch qadamdan keyin holat ===
  opt.state uzunligi: 4
  kalit             shakl        norma
  step            skalyar          3.0
  exp_avg          (8, 4)     0.044235
  exp_avg_sq       (8, 4)     0.000034
  exp_avg = m, exp_avg_sq = s (20.7-dars)

=== 4. zero_grad set_to_none ===
  zero_grad() dan keyin grad: None
  set_to_none=False bilan grad normasi: 0.0
  None - xotira tejaladi va 'grad yo'q' aniq ko'rinadi

=== 5. Qo'lda Adam bilan solishtirish ===
  torch.optim.Adam: [ 0.177493 -0.12581   0.037581  0.139454  0.013764]
  qo'lda 20.7-bob:    [ 0.177493 -0.12581   0.037581  0.139454  0.013764]
  bir xilmi: True
  ⭐ torch.optim - bizning formulalar, boshqa hech narsa

"""Turli lr, bias/norm uchun decay yo'q (real torch)."""

import torch
import torch.nn as nn


class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.asos = nn.Sequential(nn.Linear(10, 32), nn.LayerNorm(32),
                                  nn.ReLU(), nn.Linear(32, 32))
        self.bosh = nn.Linear(32, 3)

    def forward(self, x):
        return self.bosh(torch.relu(self.asos(x)))


def main() -> None:
    torch.manual_seed(0)
    model = Model()

    print("=== 1. Parametrlarni tasniflash ===")
    print(f"  {'nom':<18} {'shakl':>10} {'ndim':>5} {'decay':>7}")
    for nom, p in model.named_parameters():
        print(f"  {nom:<18} {str(tuple(p.shape)):>10} {p.ndim:>5} "
              f"{'ha' if p.ndim >= 2 else 'yo_q':>7}")

    print("\n=== 2. Guruhlar ===")
    decay = [p for p in model.parameters() if p.ndim >= 2]
    decaysiz = [p for p in model.parameters() if p.ndim < 2]
    opt = torch.optim.AdamW([
        {"params": decay, "weight_decay": 0.1},
        {"params": decaysiz, "weight_decay": 0.0},
    ], lr=1e-2)
    for i, g in enumerate(opt.param_groups):
        soni = sum(p.numel() for p in g["params"])
        print(f"  guruh {i}: {len(g['params'])} tensor, {soni} son, "
              f"wd={g['weight_decay']}, lr={g['lr']}")

    print("\n=== 3. Decay bias ga nima qiladi ===")
    for wd_bias in [0.0, 0.5]:
        torch.manual_seed(0)
        q = nn.Linear(1, 1)
        with torch.no_grad():
            q.bias.fill_(5.0)
        o = torch.optim.AdamW([
            {"params": [q.weight], "weight_decay": 0.0},
            {"params": [q.bias], "weight_decay": wd_bias}], lr=0.05)
        X = torch.randn(64, 1)
        y = X * 2 + 5.0
        for _ in range(300):
            o.zero_grad()
            ((q(X) - y) ** 2).mean().backward()
            o.step()
        print(f"  bias wd={wd_bias}: bias = {q.bias.item():.4f} "
              f"(haqiqiy 5.0)")
    print("  decay bias ni haqiqiy qiymatdan uzoqlashtiradi")

    print("\n=== 4. Transfer learning: asos va bosh ===")
    model = Model()
    opt = torch.optim.AdamW([
        {"params": model.asos.parameters(), "lr": 1e-4},
        {"params": model.bosh.parameters(), "lr": 1e-2},
    ], weight_decay=0.01)
    eski_asos = model.asos[0].weight.detach().clone()
    eski_bosh = model.bosh.weight.detach().clone()
    X = torch.randn(64, 10)
    y = torch.randint(0, 3, (64,))
    for _ in range(20):
        opt.zero_grad()
        nn.CrossEntropyLoss()(model(X), y).backward()
        opt.step()
    print(f"  {'qism':<8} {'lr':>8} {'o_zgarish normasi':>19}")
    print(f"  {'asos':<8} {1e-4:>8.0e} "
          f"{(model.asos[0].weight - eski_asos).norm().item():>19.6f}")
    print(f"  {'bosh':<8} {1e-2:>8.0e} "
          f"{(model.bosh.weight - eski_bosh).norm().item():>19.6f}")

    print("\n=== 5. Guruh lr ni o'zgartirish ===")
    for g in opt.param_groups:
        g["lr"] *= 0.5
    print(f"  yangi lr lar: "
          f"{[g['lr'] for g in opt.param_groups]}")

    print("\n=== 6. Bir parametr ikki guruhda ===")
    try:
        torch.optim.SGD([{"params": model.bosh.parameters()},
                         {"params": model.parameters()}], lr=0.1)
        print("  xato chiqmadi (kutilmagan)")
    except ValueError as xato:
        print(f"  ValueError: {str(xato)[:60]}")
    print("  ⭐ Har parametr faqat BITTA guruhda bo'lishi kerak")


if __name__ == "__main__":
    main()

=== 1. Parametrlarni tasniflash ===
  nom                     shakl  ndim   decay
  asos.0.weight        (32, 10)     2      ha
  asos.0.bias             (32,)     1    yo_q
  asos.1.weight           (32,)     1    yo_q
  asos.1.bias             (32,)     1    yo_q
  asos.3.weight        (32, 32)     2      ha
  asos.3.bias             (32,)     1    yo_q
  bosh.weight           (3, 32)     2      ha
  bosh.bias                (3,)     1    yo_q

=== 2. Guruhlar ===
  guruh 0: 3 tensor, 1440 son, wd=0.1, lr=0.01
  guruh 1: 5 tensor, 131 son, wd=0.0, lr=0.01

=== 3. Decay bias ga nima qiladi ===
  bias wd=0.0: bias = 5.0000 (haqiqiy 5.0)
  bias wd=0.5: bias = 2.2854 (haqiqiy 5.0)
  decay bias ni haqiqiy qiymatdan uzoqlashtiradi

=== 4. Transfer learning: asos va bosh ===
  qism           lr   o_zgarish normasi
  asos        1e-04            0.028947
  bosh        1e-02            1.526546

=== 5. Guruh lr ni o'zgartirish ===
  yangi lr lar: [5e-05, 0.005]

=== 6. Bir parametr ikki guruhda ===
  ValueError: some parameters appear in more than one parameter group
  ⭐ Har parametr faqat BITTA guruhda bo'lishi kerak

"""Jadval shakllari va noto'g'ri qadamlash xatosi (real torch)."""

import torch
import torch.nn as nn
from torch.optim import lr_scheduler as ls


def lr_tarixi(yasovchi, qadamlar):
    p = nn.Parameter(torch.zeros(1))
    opt = torch.optim.SGD([p], lr=0.1)
    sched = yasovchi(opt)
    tarix = []
    for _ in range(qadamlar):
        tarix.append(opt.param_groups[0]["lr"])
        opt.step()
        sched.step()
    return tarix


def main() -> None:
    D = 20

    print("=== 1. Jadvallar shakli (20 davr) ===")
    jadvallar = {
        "StepLR(7, 0.5)": lambda o: ls.StepLR(o, 7, 0.5),
        "Exponential(0.85)": lambda o: ls.ExponentialLR(o, 0.85),
        "Cosine(T=20)": lambda o: ls.CosineAnnealingLR(o, T_max=D),
        "Warmup+Cosine": lambda o: ls.SequentialLR(
            o, [ls.LinearLR(o, start_factor=0.1, total_iters=4),
                ls.CosineAnnealingLR(o, T_max=D - 4)], milestones=[4]),
    }
    tarixlar = {nom: lr_tarixi(f, D) for nom, f in jadvallar.items()}
    print(f"  {'davr':>5}", end="")
    for nom in jadvallar:
        print(f" {nom:>18}", end="")
    print()
    for d in [0, 2, 4, 7, 10, 15, 19]:
        print(f"  {d:>5}", end="")
        for nom in jadvallar:
            print(f" {tarixlar[nom][d]:>18.5f}", end="")
        print()

    print("\n=== 2. Noto'g'ri birlikda qadamlash ===")
    batchlar = 50
    p = nn.Parameter(torch.zeros(1))
    opt = torch.optim.SGD([p], lr=0.1)
    sched = ls.CosineAnnealingLR(opt, T_max=D)      # DAVR uchun
    davr_lr = []
    for davr in range(D):
        davr_lr.append(opt.param_groups[0]["lr"])
        for _ in range(batchlar):
            opt.step()
            sched.step()                             # ⚠️ har BATCH
    print(f"  T_max = {D} davr, lekin step() har batchda")
    print(f"  {'davr':>5} {'lr':>12}")
    for d in [0, 1, 2, 5, 10, 19]:
        print(f"  {d:>5} {davr_lr[d]:>12.6f}")
    print("  kosinus 20 BATCH da tugab, qaytadan tebranadi")

    print("\n=== 3. OneCycleLR - qadam birligida ===")
    p = nn.Parameter(torch.zeros(1))
    opt = torch.optim.SGD([p], lr=0.1)
    jami = D * batchlar
    sched = ls.OneCycleLR(opt, max_lr=0.1, total_steps=jami)
    lr_lar = []
    for _ in range(jami):
        lr_lar.append(opt.param_groups[0]["lr"])
        opt.step()
        sched.step()
    print(f"  total_steps = {jami} (davr x batch)")
    for q in [0, 150, 300, 600, 999]:
        print(f"  qadam {q:>4}: lr = {lr_lar[q]:.6f}")
    print(f"  maksimum qadam: {max(range(jami), key=lambda i: lr_lar[i])}")

    print("\n=== 4. ReduceLROnPlateau ===")
    p = nn.Parameter(torch.zeros(1))
    opt = torch.optim.SGD([p], lr=0.1)
    sched = ls.ReduceLROnPlateau(opt, mode="min", factor=0.5,
                                 patience=2)
    val_losslar = [1.0, 0.8, 0.7, 0.71, 0.72, 0.70, 0.705, 0.71, 0.72,
                   0.69, 0.695, 0.70, 0.70]
    print(f"  {'davr':>5} {'val loss':>10} {'lr':>10}")
    for d, vl in enumerate(val_losslar):
        opt.step()
        sched.step(vl)
        print(f"  {d:>5} {vl:>10.3f} "
              f"{opt.param_groups[0]['lr']:>10.4f}")
    print("  ⭐ yaxshilanish to'xtaganda lr avtomatik kamayadi")


if __name__ == "__main__":
    main()

=== 1. Jadvallar shakli (20 davr) ===
   davr     StepLR(7, 0.5)  Exponential(0.85)       Cosine(T=20)      Warmup+Cosine
      0            0.10000            0.10000            0.10000            0.01000
      2            0.10000            0.07225            0.09755            0.05500
      4            0.10000            0.05220            0.09045            0.10000
      7            0.05000            0.03206            0.07270            0.09157
     10            0.05000            0.01969            0.05000            0.06913
     15            0.02500            0.00874            0.01464            0.02222
     19            0.02500            0.00456            0.00062            0.00096

=== 2. Noto'g'ri birlikda qadamlash ===
  T_max = 20 davr, lekin step() har batchda
   davr           lr
      0     0.100000
      1     0.050000
      2     0.000000
      5     0.050000
     10     0.000000
     19     0.050000
  kosinus 20 BATCH da tugab, qaytadan tebranadi

=== 3. OneCycleLR - qadam birligida ===
  total_steps = 1000 (davr x batch)
  qadam    0: lr = 0.004000
  qadam  150: lr = 0.052252
  qadam  300: lr = 0.099999
  qadam  600: lr = 0.060907
  qadam  999: lr = 0.000000
  maksimum qadam: 299

=== 4. ReduceLROnPlateau ===
   davr   val loss         lr
      0      1.000     0.1000
      1      0.800     0.1000
      2      0.700     0.1000
      3      0.710     0.1000
      4      0.720     0.1000
      5      0.700     0.0500
      6      0.705     0.0500
      7      0.710     0.0500
      8      0.720     0.0250
      9      0.690     0.0250
     10      0.695     0.0250
     11      0.700     0.0250
     12      0.700     0.0125
  ⭐ yaxshilanish to'xtaganda lr avtomatik kamayadi

"""Gradient clipping va o'rgatishni to'xtatib davom ettirish."""

import shutil
import tempfile
from pathlib import Path

import torch
import torch.nn as nn


def yasa(seed=0):
    torch.manual_seed(seed)
    model = nn.Sequential(nn.Linear(8, 32), nn.ReLU(),
                          nn.Linear(32, 32), nn.ReLU(),
                          nn.Linear(32, 1))
    opt = torch.optim.AdamW(model.parameters(), lr=0.01)
    sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=40)
    return model, opt, sched


def davr(model, opt, X, y, clip=None):
    model.train()
    normalar = []
    for i in range(0, len(X), 32):
        opt.zero_grad()
        loss = ((model(X[i:i + 32]) - y[i:i + 32]) ** 2).mean()
        loss.backward()
        if clip is not None:
            normalar.append(float(torch.nn.utils.clip_grad_norm_(
                model.parameters(), clip)))
        opt.step()
    return loss.item(), normalar


def main() -> None:
    g = torch.Generator().manual_seed(0)
    X = torch.randn(512, 8, generator=g)
    y = (X[:, :1] * 3 + X[:, 1:2] ** 2
         + 0.1 * torch.randn(512, 1, generator=g))

    print("=== 1. clip_grad_norm_ nimani qaytaradi ===")
    model, opt, _ = yasa()
    y_katta = y * 100                   # ataylab katta masshtab
    _, normalar = davr(model, opt, X, y_katta, clip=1.0)
    print(f"  clipping OLDIDAGI normalar (birinchi 5): "
          f"{[round(n, 1) for n in normalar[:5]]}")
    umumiy = torch.sqrt(sum(p.grad.norm() ** 2
                            for p in model.parameters()))
    print(f"  oxirgi qadamdan keyin haqiqiy norma: {umumiy.item():.4f}")
    print("  norma <= 1.0 ga cheklandi, yo'nalish saqlandi")

    print("\n=== 2. Clipping beqarorlikni to'xtatadi ===")
    print(f"  {'variant':<14} {'1-davr':>12} {'5-davr':>12}")
    for nom, clip in [("clipping siz", None), ("clip=1.0", 1.0)]:
        torch.manual_seed(0)
        m = nn.Sequential(nn.Linear(8, 32), nn.ReLU(), nn.Linear(32, 1))
        o = torch.optim.SGD(m.parameters(), lr=0.05)
        loss_lar = []
        for _ in range(5):
            l, _ = davr(m, o, X, y_katta, clip=clip)
            loss_lar.append(l)
        print(f"  {nom:<14} {loss_lar[0]:>12.3e} {loss_lar[-1]:>12.3e}")

    papka = Path(tempfile.mkdtemp(prefix="torch_ckpt_"))
    try:
        print("\n=== 3. 40 davr uzluksiz ===")
        model, opt, sched = yasa()
        for d in range(40):
            l, _ = davr(model, opt, X, y)
            sched.step()
        uzluksiz = model(X).detach()
        print(f"  yakuniy loss: {l:.6f}")

        print("\n=== 4. 20 davr + checkpoint + 20 davr ===")
        model, opt, sched = yasa()
        for d in range(20):
            davr(model, opt, X, y)
            sched.step()
        yol = papka / "ckpt.pt"
        torch.save({"model": model.state_dict(),
                    "opt": opt.state_dict(),
                    "sched": sched.state_dict(),
                    "davr": 19}, yol)
        del model, opt, sched
        model, opt, sched = yasa(seed=123)      # boshqa boshlanish
        paket = torch.load(yol, weights_only=True)
        model.load_state_dict(paket["model"])
        opt.load_state_dict(paket["opt"])
        sched.load_state_dict(paket["sched"])
        for d in range(paket["davr"] + 1, 40):
            l, _ = davr(model, opt, X, y)
            sched.step()
        davomli = model(X).detach()
        print(f"  yakuniy loss: {l:.6f}")
        print(f"  uzluksiz bilan bir xilmi: "
              f"{torch.allclose(uzluksiz, davomli, atol=1e-6)}")

        print("\n=== 5. Optimizator holatini saqlamasak ===")
        model, opt, sched = yasa()
        for d in range(20):
            davr(model, opt, X, y)
            sched.step()
        faqat_model = {k: v.clone() for k, v in
                       model.state_dict().items()}
        model2, opt2, sched2 = yasa(seed=123)
        model2.load_state_dict(faqat_model)
        oldin = ((model2(X) - y) ** 2).mean().item()
        l2, _ = davr(model2, opt2, X, y)
        print(f"  yuklashdan oldin loss: {oldin:.6f}")
        print(f"  bir davrdan keyin:     {l2:.6f}")
        print(f"  lr: {opt2.param_groups[0]['lr']:.5f} "
              f"(to'g'risi {sched.get_last_lr()[0]:.5f})")
        print("  optimizator 'noldan' boshladi - lr va momentlar yo'qoldi")
        print("  ⭐ Checkpoint: model + opt + sched + davr")
    finally:
        shutil.rmtree(papka, ignore_errors=True)


if __name__ == "__main__":
    main()

=== 1. clip_grad_norm_ nimani qaytaradi ===
  clipping OLDIDAGI normalar (birinchi 5): [175.7, 722.6, 262.6, 310.5, 431.1]
  oxirgi qadamdan keyin haqiqiy norma: 1.0000
  norma <= 1.0 ga cheklandi, yo'nalish saqlandi

=== 2. Clipping beqarorlikni to'xtatadi ===
  variant              1-davr       5-davr
  clipping siz            nan          nan
  clip=1.0          1.126e+05    1.098e+05

=== 3. 40 davr uzluksiz ===
  yakuniy loss: 0.019614

=== 4. 20 davr + checkpoint + 20 davr ===
  yakuniy loss: 0.019614
  uzluksiz bilan bir xilmi: True

=== 5. Optimizator holatini saqlamasak ===
  yuklashdan oldin loss: 0.048213
  bir davrdan keyin:     0.072430
  lr: 0.01000 (to'g'risi 0.00500)
  optimizator 'noldan' boshladi - lr va momentlar yo'qoldi
  ⭐ Checkpoint: model + opt + sched + davr

CosineAnnealingLR(opt, T_max=davrlar)  # + step() har batchda   # ⚠️
# T_max=davrlar bo'lsa step() DAVR oxirida                     # ✅

sched.step(); opt.step()                          # ⚠️
opt.step(); sched.step()                          # ✅

AdamW(model.parameters(), weight_decay=0.1)       # ⚠️
AdamW(guruhlar(model, 0.1))                       # ✅ p.ndim < 2 ajratilgan

clip_grad_norm_(...); loss.backward()             # ⚠️
loss.backward(); clip_grad_norm_(...); opt.step() # ✅

torch.save(model.state_dict(), yol)               # ⚠️ davom ettirish uchun
torch.save({"model":..., "opt":..., "sched":...}) # ✅

ReduceLROnPlateau(opt).step()                     # ⚠️
sched.step(val_loss)                              # ✅

opt = AdamW(eski_model.parameters()); model = Yangi()   # ⚠️
model = Yangi(); opt = AdamW(model.parameters())        # ✅

1.  # opt.state da Adam nima saqlaydi?
2.  # param_groups nima?
3.  # qaysi parametrlarga decay qo'yilmaydi?
4.  # oddiy qoida?
5.  # sched.step() qachon?
6.  # OneCycleLR qaysi birlikda?
7.  # ReduceLROnPlateau ga nima beriladi?
8.  # clip_grad_norm_ nima qaytaradi?
9.  # clipping qayerda?
10. # checkpoint da nima?
11. # opt holati saqlanmasa?
12. # bir parametr ikki guruhda?

1.  sched.step(); opt.step()

2.  AdamW(model.parameters(), weight_decay=0.1)

3.  clip_grad_norm_(...); loss.backward()

4.  ReduceLROnPlateau(opt).step()

5.  torch.save(model.state_dict(), yol)   # davom ettirish uchun