20.6-dars: Orqaga tarqalish

BITTA O'ZGARUVCHI:
  y = f(g(x))  ->  dy/dx = f'(g(x)) * g'(x)

TARMOQDA:
  L <- ZL <- A_{L-1} <- Z_{L-1} <- ... <- A1 <- Z1 <- X

  dL/dW1 = dL/dZL * dZL/dA_{L-1} * dA_{L-1}/dZ_{L-1} * ... * dZ1/dW1

TARTIB MUHIM:
  oldindan orqaga (forward mode): har o'zgaruvchi uchun alohida o'tish
  ORQADAN oldinga (reverse mode): BITTA o'tishda HAMMASI

  N parametr uchun:
    sonli hosila:     2N ta oldinga o'tish
    orqaga tarqalish: 1 ta oldinga + 1 ta orqaga o'tish

QATLAM:  Z = A_oldingi @ W + b,   A = f(Z)

KIRISH:  dA (yuqoridan kelgan gradient, shakli A bilan bir xil)

1. dZ = dA * f'(Z)                shakl (N, d_out)
2. dW = A_oldingi.T @ dZ          shakl (d_in, d_out)  <- W bilan bir xil
3. db = dZ.sum(axis=0)            shakl (d_out,)
4. dA_oldingi = dZ @ W.T          shakl (N, d_in)      <- pastga uzatiladi

SHAKL — ENG YAXSHI TEKSHIRUV:
  dW shakli W shakli bilan AYNAN bir xil bo'lishi SHART
  aks holda transpozitsiya xato

LOSS DAN BIRINCHI GRADIENT:

MSE (chiqish chiziqli):
  L = mean((A - Y)^2)
  dL/dZL = 2*(A - Y) / N

CROSS-ENTROPY + SOFTMAX (birgalikda):
  L = -mean(log softmax(ZL)[y])
  dL/dZL = (P - Y_onehot) / N          <- softmax hosilasi QISQARDI

BINARY CE + SIGMOID:
  dL/dZL = (p - y) / N

⭐ Shuning uchun softmax/sigmoid ni loss ICHIDA qoldirish kerak:
   formula soddalashadi va barqaror bo'ladi

MARKAZIY FARQ:
  df/dw ~ (f(w + h) - f(w - h)) / (2h)

  xatolik O(h^2)  (oddiy farqda O(h))

h TANLASH:
  juda katta -> approksimatsiya xatosi
  juda kichik -> yaxlitlash xatosi
  float64 uchun h = 1e-5 .. 1e-7 optimal

NISBIY XATO:
  |analitik - sonli| / max(|analitik|, |sonli|, 1e-12)

  < 1e-7   ajoyib
  < 1e-5   yaxshi
  > 1e-3   XATO BOR

DIQQAT: ReLU ning burilish nuqtasida (z ~ 0) sonli hosila
        to'g'ri bo'lmaydi - bu XATO EMAS

OLDINGA O'TISH: N * d_in * d_out ta amal (har qatlamda)
ORQAGA O'TISH:  ~2 barobar ko'p
  dW:           d_in * N * d_out
  dA_oldingi:   N * d_out * d_in

JAMI: orqaga tarqalish oldinga o'tishdan ~2x qimmat
      bitta o'rgatish qadami ~3x inference

XOTIRA: barcha A va Z saqlanishi kerak
  chuqur tarmoqda bu asosiy cheklov
  YECHIM: gradient checkpointing (qayta hisoblash)

SOG'LOM: har qatlamda gradient normasi bir tartibda

MUAMMOLAR:
  YO'QOLAYOTGAN: pastki qatlamlarda ~0  -> o'rganmaydi
    sabab: to'yingan aktivatsiya, kichik og'irlik
  PORTLAYOTGAN: normalar ~1e6  -> NaN
    sabab: katta og'irlik, katta lr, chuqur tarmoq

DIAGNOSTIKA:
  for i, dW in enumerate(gradientlar):
      print(i, np.linalg.norm(dW))

YECHIMLAR:
  to'g'ri boshlash (He/Xavier)
  gradient clipping: g = g * min(1, max_norm / ||g||)
  BatchNorm, residual ulanishlar

import numpy as np

def orqaga(kesh, Ws, Y):
    """Cross-entropy + softmax chiqish uchun gradientlar."""
    A, Z = kesh["A"], kesh["Z"]
    N = len(Y)
    P = softmax(Z[-1])
    dZ = (P - Y) / N                       # boshlang'ich gradient
    dWs, dbs = [None] * len(Ws), [None] * len(Ws)
    for i in range(len(Ws) - 1, -1, -1):
        dWs[i] = A[i].T @ dZ               # (d_in, d_out)
        dbs[i] = dZ.sum(axis=0)            # (d_out,)
        if i > 0:
            dA = dZ @ Ws[i].T              # (N, d_in)
            dZ = dA * (Z[i - 1] > 0)       # ReLU hosilasi
    return dWs, dbs

# sonli tekshiruv
def sonli(f, w, i, j, h=1e-6):
    w[i, j] += h; f1 = f()
    w[i, j] -= 2 * h; f2 = f()
    w[i, j] += h
    return (f1 - f2) / (2 * h)

dZ = dA * f'(Z)
dW = A_oldingi.T @ dZ
db = dZ.sum(axis=0)
dA_oldingi = dZ @ W.T
shakl tekshiruvi: dW.shape == W.shape

"""Kichik tarmoqda har gradientni ochiq hisoblash (real numpy)."""

import numpy as np


def main() -> None:
    rng = np.random.default_rng(0)
    N, d0, d1, K = 4, 3, 5, 2
    X = rng.normal(0, 1, (N, d0))
    y = np.array([0, 1, 1, 0])
    Y = np.zeros((N, K))
    Y[np.arange(N), y] = 1.0
    W1 = rng.normal(0, np.sqrt(2 / d0), (d0, d1))
    b1 = np.zeros(d1)
    W2 = rng.normal(0, np.sqrt(2 / d1), (d1, K))
    b2 = np.zeros(K)

    print("=== 1. Oldinga o'tish ===")
    Z1 = X @ W1 + b1
    A1 = np.maximum(0, Z1)
    Z2 = A1 @ W2 + b2
    Zs = Z2 - Z2.max(axis=1, keepdims=True)
    P = np.exp(Zs) / np.exp(Zs).sum(axis=1, keepdims=True)
    loss = -np.mean(np.log(P[np.arange(N), y]))
    print(f"  {'nom':>5} {'shakl':>10}")
    for nom, m in [("X", X), ("Z1", Z1), ("A1", A1), ("Z2", Z2),
                   ("P", P)]:
        print(f"  {nom:>5} {str(m.shape):>10}")
    print(f"  loss: {loss:.6f}")

    print("\n=== 2. Boshlang'ich gradient ===")
    dZ2 = (P - Y) / N
    print(f"  dZ2 = (P - Y) / N, shakl {dZ2.shape}")
    print(f"  {'namuna':>7} {'P':<20} {'Y':<12} {'dZ2':<20}")
    for i in range(N):
        print(f"  {i:>7} {str(P[i].round(4)):<20} "
              f"{str(Y[i].astype(int)):<12} {str(dZ2[i].round(4)):<20}")

    print("\n=== 3. Ikkinchi qatlam gradientlari ===")
    dW2 = A1.T @ dZ2
    db2 = dZ2.sum(axis=0)
    print(f"  dW2 = A1.T @ dZ2:  {A1.T.shape} @ {dZ2.shape} "
          f"-> {dW2.shape}")
    print(f"  W2 shakli:         {W2.shape}   mosmi: "
          f"{dW2.shape == W2.shape}")
    print(f"  db2 = dZ2.sum(0):  {db2.shape}  mosmi: "
          f"{db2.shape == b2.shape}")
    print(f"  dW2 =\n{dW2.round(5)}")

    print("\n=== 4. Pastga uzatish ===")
    dA1 = dZ2 @ W2.T
    dZ1 = dA1 * (Z1 > 0)
    print(f"  dA1 = dZ2 @ W2.T:  {dZ2.shape} @ {W2.T.shape} "
          f"-> {dA1.shape}")
    print(f"  dZ1 = dA1 * (Z1 > 0):  {dZ1.shape}")
    print(f"  {'namuna':>7} {'faol neyron':>13} {'nolga aylangan':>16}")
    for i in range(N):
        faol = int((Z1[i] > 0).sum())
        print(f"  {i:>7} {faol:>13} {d1 - faol:>16}")

    print("\n=== 5. Birinchi qatlam gradientlari ===")
    dW1 = X.T @ dZ1
    db1 = dZ1.sum(axis=0)
    print(f"  dW1 shakli: {dW1.shape}, W1 shakli: {W1.shape}, "
          f"mosmi: {dW1.shape == W1.shape}")
    print(f"  db1 shakli: {db1.shape}, b1 shakli: {b1.shape}")

    print("\n=== 6. Gradient normalari ===")
    print(f"  {'parametr':<8} {'shakl':>10} {'norma':>12} "
          f"{'maksimal |g|':>14}")
    for nom, g in [("dW1", dW1), ("db1", db1), ("dW2", dW2),
                   ("db2", db2)]:
        print(f"  {nom:<8} {str(g.shape):>10} "
              f"{np.linalg.norm(g):>12.6f} {np.abs(g).max():>14.6f}")
    print("  ⭐ Har qatlamda uchta formula - boshqa hech narsa yo'q")


if __name__ == "__main__":
    main()

=== 1. Oldinga o'tish ===
    nom      shakl
      X     (4, 3)
     Z1     (4, 5)
     A1     (4, 5)
     Z2     (4, 2)
      P     (4, 2)
  loss: 0.848459

=== 2. Boshlang'ich gradient ===
  dZ2 = (P - Y) / N, shakl (4, 2)
   namuna P                    Y            dZ2
        0 [0.5209 0.4791]      [1 0]        [-0.1198  0.1198]
        1 [0.5011 0.4989]      [0 1]        [ 0.1253 -0.1253]
        2 [0.6404 0.3596]      [0 1]        [ 0.1601 -0.1601]
        3 [0.3594 0.6406]      [1 0]        [-0.1602  0.1602]

=== 3. Ikkinchi qatlam gradientlari ===
  dW2 = A1.T @ dZ2:  (5, 4) @ (4, 2) -> (5, 2)
  W2 shakli:         (5, 2)   mosmi: True
  db2 = dZ2.sum(0):  (2,)  mosmi: True
  dW2 =
[[-0.40693  0.40693]
 [-0.01856  0.01856]
 [-0.15388  0.15388]
 [-0.1295   0.1295 ]
 [ 0.1448  -0.1448 ]]

=== 4. Pastga uzatish ===
  dA1 = dZ2 @ W2.T:  (4, 2) @ (2, 5) -> (4, 5)
  dZ1 = dA1 * (Z1 > 0):  (4, 5)
   namuna   faol neyron   nolga aylangan
        0             2                3
        1             1                4
        2             1                4
        3             4                1

=== 5. Birinchi qatlam gradientlari ===
  dW1 shakli: (3, 5), W1 shakli: (3, 5), mosmi: True
  db1 shakli: (5,), b1 shakli: (5,)

=== 6. Gradient normalari ===
  parametr      shakl        norma   maksimal |g|
  dW1          (3, 5)     0.269126       0.145923
  db1            (5,)     0.245338       0.217734
  dW2          (5, 2)     0.674317       0.406928
  db2            (2,)     0.007704       0.005448
  ⭐ Har qatlamda uchta formula - boshqa hech narsa yo'q

"""Analitik gradientni sonli hosila bilan solishtirish (real numpy)."""

import numpy as np


def softmax(Z):
    Z = Z - Z.max(axis=1, keepdims=True)
    E = np.exp(Z)
    return E / E.sum(axis=1, keepdims=True)


class Tarmoq:
    def __init__(self, olchamlar, seed=0):
        rng = np.random.default_rng(seed)
        self.Ws = [rng.normal(0, np.sqrt(2 / a), (a, b))
                   for a, b in zip(olchamlar[:-1], olchamlar[1:])]
        self.bs = [np.zeros(b) for b in olchamlar[1:]]

    def loss(self, X, y):
        A = X
        for i, (W, b) in enumerate(zip(self.Ws, self.bs)):
            Z = A @ W + b
            A = np.maximum(0, Z) if i < len(self.Ws) - 1 else Z
        P = softmax(A)
        return -np.mean(np.log(np.clip(P[np.arange(len(y)), y],
                                       1e-15, None)))

    def gradientlar(self, X, y):
        N = len(y)
        A_lar, Z_lar = [X], []
        A = X
        for i, (W, b) in enumerate(zip(self.Ws, self.bs)):
            Z = A @ W + b
            Z_lar.append(Z)
            A = np.maximum(0, Z) if i < len(self.Ws) - 1 else Z
            A_lar.append(A)
        Y = np.zeros_like(A)
        Y[np.arange(N), y] = 1.0
        dZ = (softmax(A) - Y) / N
        dWs = [None] * len(self.Ws)
        dbs = [None] * len(self.bs)
        for i in range(len(self.Ws) - 1, -1, -1):
            dWs[i] = A_lar[i].T @ dZ
            dbs[i] = dZ.sum(axis=0)
            if i > 0:
                dZ = (dZ @ self.Ws[i].T) * (Z_lar[i - 1] > 0)
        return dWs, dbs


def nisbiy_xato(a, s):
    return abs(a - s) / max(abs(a), abs(s), 1e-12)


def main() -> None:
    rng = np.random.default_rng(1)
    X = rng.normal(0, 1, (12, 6))
    y = rng.integers(0, 3, 12)
    t = Tarmoq([6, 8, 5, 3])
    dWs, dbs = t.gradientlar(X, y)

    print("=== 1. Shakllar mosmi ===")
    print(f"  {'parametr':<6} {'shakl':>10} {'gradient':>10} {'mos':>6}")
    for i, (W, dW) in enumerate(zip(t.Ws, dWs)):
        print(f"  {f'W{i + 1}':<6} {str(W.shape):>10} "
              f"{str(dW.shape):>10} {str(W.shape == dW.shape):>6}")
    for i, (b, db) in enumerate(zip(t.bs, dbs)):
        print(f"  {f'b{i + 1}':<6} {str(b.shape):>10} "
              f"{str(db.shape):>10} {str(b.shape == db.shape):>6}")

    print("\n=== 2. Sonli tekshiruv: W ===")
    h = 1e-6
    print(f"  {'parametr':<10} {'analitik':>13} {'sonli':>13} "
          f"{'nisbiy xato':>13}")
    for qi in range(3):
        W = t.Ws[qi]
        i, j = int(rng.integers(W.shape[0])), int(rng.integers(W.shape[1]))
        asl = W[i, j]
        W[i, j] = asl + h
        l1 = t.loss(X, y)
        W[i, j] = asl - h
        l2 = t.loss(X, y)
        W[i, j] = asl
        sonli = (l1 - l2) / (2 * h)
        a = dWs[qi][i, j]
        print(f"  {f'W{qi + 1}[{i},{j}]':<10} {a:>13.8f} {sonli:>13.8f} "
              f"{nisbiy_xato(a, sonli):>13.2e}")

    print("\n=== 3. Sonli tekshiruv: b ===")
    print(f"  {'parametr':<10} {'analitik':>13} {'sonli':>13} "
          f"{'nisbiy xato':>13}")
    for qi in range(3):
        b = t.bs[qi]
        j = int(rng.integers(b.shape[0]))
        asl = b[j]
        b[j] = asl + h
        l1 = t.loss(X, y)
        b[j] = asl - h
        l2 = t.loss(X, y)
        b[j] = asl
        sonli = (l1 - l2) / (2 * h)
        a = dbs[qi][j]
        print(f"  {f'b{qi + 1}[{j}]':<10} {a:>13.8f} {sonli:>13.8f} "
              f"{nisbiy_xato(a, sonli):>13.2e}")

    print("\n=== 4. h ning tanlanishi ===")
    W = t.Ws[0]
    i, j = 2, 3
    asl = W[i, j]
    a = dWs[0][i, j]
    print(f"  {'h':>10} {'sonli':>14} {'nisbiy xato':>14}")
    for hh in [1e-2, 1e-4, 1e-6, 1e-8, 1e-10]:
        W[i, j] = asl + hh
        l1 = t.loss(X, y)
        W[i, j] = asl - hh
        l2 = t.loss(X, y)
        W[i, j] = asl
        s = (l1 - l2) / (2 * hh)
        print(f"  {hh:>10.0e} {s:>14.9f} {nisbiy_xato(a, s):>14.2e}")
    print("  ⭐ h ~ 1e-6 optimal: kichikroq -> yaxlitlash xatosi")

    print("\n=== 5. Ataylab buzilgan gradient ===")
    buzilgan = [dW.T.copy() if dW.shape[0] == dW.shape[1] else dW * 2
                for dW in dWs]
    print(f"  {'parametr':<10} {'to_g_ri':>13} {'buzilgan':>13} "
          f"{'nisbiy xato':>13}")
    for qi in range(3):
        W = t.Ws[qi]
        i, j = 0, 0
        asl = W[i, j]
        W[i, j] = asl + h
        l1 = t.loss(X, y)
        W[i, j] = asl - h
        l2 = t.loss(X, y)
        W[i, j] = asl
        s = (l1 - l2) / (2 * h)
        print(f"  {f'W{qi + 1}[0,0]':<10} {dWs[qi][0, 0]:>13.8f} "
              f"{buzilgan[qi][0, 0]:>13.8f} "
              f"{nisbiy_xato(buzilgan[qi][0, 0], s):>13.2e}")
    print("  buzilgan gradientda nisbiy xato KATTA - tekshiruv ishlaydi")


if __name__ == "__main__":
    main()

=== 1. Shakllar mosmi ===
  parametr      shakl   gradient    mos
  W1         (6, 8)     (6, 8)   True
  W2         (8, 5)     (8, 5)   True
  W3         (5, 3)     (5, 3)   True
  b1           (8,)       (8,)   True
  b2           (5,)       (5,)   True
  b3           (3,)       (3,)   True

=== 2. Sonli tekshiruv: W ===
  parametr        analitik         sonli   nisbiy xato
  W1[2,7]      -0.00532963   -0.00532963      2.82e-08
  W2[0,3]       0.00246373    0.00246373      7.23e-08
  W3[1,0]       0.04977066    0.04977066      4.79e-09

=== 3. Sonli tekshiruv: b ===
  parametr        analitik         sonli   nisbiy xato
  b1[5]        -0.09992326   -0.09992326      9.65e-10
  b2[3]         0.06553159    0.06553159      3.01e-10
  b3[2]        -0.37234180   -0.37234180      9.56e-11

=== 4. h ning tanlanishi ===
           h          sonli    nisbiy xato
       1e-02    0.028577505       1.03e-07
       1e-04    0.028577508       2.50e-11
       1e-06    0.028577508       2.55e-09
       1e-08    0.028577518       3.64e-07
       1e-10    0.028578251       2.60e-05
  ⭐ h ~ 1e-6 optimal: kichikroq -> yaxlitlash xatosi

=== 5. Ataylab buzilgan gradient ===
  parametr         to_g_ri      buzilgan   nisbiy xato
  W1[0,0]      -0.27137715   -0.54275431      5.00e-01
  W2[0,0]       0.09459554    0.18919107      5.00e-01
  W3[0,0]       0.05069598    0.10139196      5.00e-01
  buzilgan gradientda nisbiy xato KATTA - tekshiruv ishlaydi

"""Qatlamlar bo'ylab gradient qanday o'zgaradi (real numpy)."""

import numpy as np


def softmax(Z):
    Z = Z - Z.max(axis=1, keepdims=True)
    E = np.exp(Z)
    return E / E.sum(axis=1, keepdims=True)


def gradient_normalari(olchamlar, masshtab, aktiv="relu", seed=0):
    rng = np.random.default_rng(seed)
    Ws = [rng.normal(0, masshtab(a), (a, b))
          for a, b in zip(olchamlar[:-1], olchamlar[1:])]
    N = 128
    X = rng.normal(0, 1, (N, olchamlar[0]))
    y = rng.integers(0, olchamlar[-1], N)
    A_lar, Z_lar = [X], []
    A = X
    for i, W in enumerate(Ws):
        Z = A @ W
        Z_lar.append(Z)
        if i < len(Ws) - 1:
            A = np.maximum(0, Z) if aktiv == "relu" else np.tanh(Z)
        else:
            A = Z
        A_lar.append(A)
    Y = np.zeros_like(A)
    Y[np.arange(N), y] = 1.0
    dZ = (softmax(A) - Y) / N
    normalar = []
    for i in range(len(Ws) - 1, -1, -1):
        normalar.append(float(np.linalg.norm(A_lar[i].T @ dZ)))
        if i > 0:
            dA = dZ @ Ws[i].T
            dZ = dA * ((Z_lar[i - 1] > 0) if aktiv == "relu"
                       else (1 - np.tanh(Z_lar[i - 1]) ** 2))
    return normalar[::-1]


def main() -> None:
    olchamlar = [64] + [64] * 9 + [4]

    print("=== 1. Boshlang'ich masshtabga qarab ===")
    variantlar = {
        "kichik 0.05-bob": lambda n: 0.05,
        "He sqrt(2/n)": lambda n: np.sqrt(2.0 / n),
        "katta 0.4-bob": lambda n: 0.4,
    }
    natijalar = {nom: gradient_normalari(olchamlar, m)
                 for nom, m in variantlar.items()}
    print(f"  {'qatlam':>7}", end="")
    for nom in variantlar:
        print(f" {nom:>16}", end="")
    print()
    for i in [0, 2, 5, 8, 9]:
        print(f"  {i + 1:>7}", end="")
        for nom in variantlar:
            print(f" {natijalar[nom][i]:>16.3e}", end="")
        print()

    print("\n=== 2. Birinchi / oxirgi nisbati ===")
    print(f"  {'masshtab':<16} {'1-qatlam':>13} {'oxirgi':>13} "
          f"{'nisbat':>13}")
    for nom, n in natijalar.items():
        print(f"  {nom:<16} {n[0]:>13.3e} {n[-1]:>13.3e} "
              f"{n[0] / n[-1]:>13.3e}")
    print("  nisbat 1 dan juda uzoq bo'lsa - muammo bor")

    print("\n=== 3. relu va tanh ===")
    print(f"  {'aktivatsiya':<10} {'1-qatlam':>13} {'oxirgi':>13} "
          f"{'nisbat':>13}")
    for aktiv in ["relu", "tanh"]:
        n = gradient_normalari(olchamlar, lambda k: np.sqrt(2.0 / k),
                               aktiv=aktiv)
        print(f"  {aktiv:<10} {n[0]:>13.3e} {n[-1]:>13.3e} "
              f"{n[0] / n[-1]:>13.3e}")

    print("\n=== 4. Chuqurlik ortganda ===")
    print(f"  {'chuqurlik':>10} {'1-qatlam normasi':>19} "
          f"{'oxirgi':>13}")
    for chuq in [3, 6, 12, 20]:
        olch = [64] + [64] * chuq + [4]
        n = gradient_normalari(olch, lambda k: 0.05)
        print(f"  {chuq:>10} {n[0]:>19.3e} {n[-1]:>13.3e}")
    print("  kichik masshtab + chuqurlik = gradient yo'qoladi")

    print("\n=== 5. Gradient clipping ===")
    rng = np.random.default_rng(3)
    gradientlar = [rng.normal(0, s, (10, 10))
                   for s in [0.01, 0.1, 1.0, 50.0]]
    max_norma = 5.0
    print(f"  {'asl norma':>12} {'koeffitsiyent':>14} "
          f"{'yangi norma':>13}")
    for g in gradientlar:
        norma = float(np.linalg.norm(g))
        koef = min(1.0, max_norma / norma)
        print(f"  {norma:>12.4f} {koef:>14.6f} {norma * koef:>13.4f}")
    print("  ⭐ Clipping yo'nalishni saqlab, uzunlikni cheklaydi")


if __name__ == "__main__":
    main()

=== 1. Boshlang'ich masshtabga qarab ===
   qatlam    kichik 0.05-bob     He sqrt(2/n)      katta 0.4-bob
        1        5.222e-06        4.685e-01        1.010e+03
        3        5.155e-06        5.252e-01        1.831e+03
        6        4.202e-06        4.521e-01        2.283e+03
        9        3.789e-06        5.269e-01        3.915e+03
       10        3.620e-06        5.281e-01        3.724e+03

=== 2. Birinchi / oxirgi nisbati ===
  masshtab              1-qatlam        oxirgi        nisbat
  kichik 0.05-bob        5.222e-06     3.620e-06     1.443e+00
  He sqrt(2/n)         4.685e-01     5.281e-01     8.871e-01
  katta 0.4-bob          1.010e+03     3.724e+03     2.712e-01
  nisbat 1 dan juda uzoq bo'lsa - muammo bor

=== 3. relu va tanh ===
  aktivatsiya      1-qatlam        oxirgi        nisbat
  relu           4.685e-01     5.281e-01     8.871e-01
  tanh           6.615e-01     5.815e-01     1.138e+00

=== 4. Chuqurlik ortganda ===
   chuqurlik    1-qatlam normasi        oxirgi
           3           1.410e-02     1.237e-02
           6           3.201e-04     3.276e-04
          12           1.140e-07     5.413e-08
          20           3.522e-12     1.841e-12
  kichik masshtab + chuqurlik = gradient yo'qoladi

=== 5. Gradient clipping ===
     asl norma  koeffitsiyent   yangi norma
        0.1068       1.000000        0.1068
        0.9930       1.000000        0.9930
        9.7705       0.511745        5.0000
      496.6871       0.010067        5.0000
  ⭐ Clipping yo'nalishni saqlab, uzunlikni cheklaydi

"""Backprop ishlayotganini o'rgatish bilan isbotlash."""

import numpy as np
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score, log_loss
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler


def softmax(Z):
    Z = Z - Z.max(axis=1, keepdims=True)
    E = np.exp(Z)
    return E / E.sum(axis=1, keepdims=True)


class Tarmoq:
    def __init__(self, olchamlar, seed=0):
        rng = np.random.default_rng(seed)
        self.Ws = [rng.normal(0, np.sqrt(2 / a), (a, b))
                   for a, b in zip(olchamlar[:-1], olchamlar[1:])]
        self.bs = [np.zeros(b) for b in olchamlar[1:]]

    def oldinga(self, X):
        A_lar, Z_lar = [X], []
        A = X
        for i, (W, b) in enumerate(zip(self.Ws, self.bs)):
            Z = A @ W + b
            Z_lar.append(Z)
            A = np.maximum(0, Z) if i < len(self.Ws) - 1 else Z
            A_lar.append(A)
        return A_lar, Z_lar

    def bashorat(self, X):
        return softmax(self.oldinga(X)[0][-1])

    def qadam(self, X, y, lr):
        N = len(y)
        A_lar, Z_lar = self.oldinga(X)
        Y = np.zeros_like(A_lar[-1])
        Y[np.arange(N), y] = 1.0
        dZ = (softmax(A_lar[-1]) - Y) / N
        for i in range(len(self.Ws) - 1, -1, -1):
            dW = A_lar[i].T @ dZ
            db = dZ.sum(axis=0)
            if i > 0:
                dZ = (dZ @ self.Ws[i].T) * (Z_lar[i - 1] > 0)
            self.Ws[i] -= lr * dW
            self.bs[i] -= lr * db


def main() -> None:
    X, y = make_classification(n_samples=4000, n_features=20,
                               n_informative=10, n_redundant=4,
                               n_classes=4, flip_y=0.08, class_sep=1.1,
                               random_state=0)
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3,
                                          random_state=0, stratify=y)
    sc = StandardScaler().fit(Xtr)
    Xtr, Xte = sc.transform(Xtr), sc.transform(Xte)

    print("=== 1. Tarmoq ===")
    t = Tarmoq([20, 64, 32, 4])
    print(f"  arxitektura: 20 -> 64 -> 32 -> 4")
    print(f"  parametrlar: "
          f"{sum(W.size for W in t.Ws) + sum(b.size for b in t.bs)}")
    print(f"  boshlang'ich test aniqlik: "
          f"{accuracy_score(yte, t.bashorat(Xte).argmax(1)):.4f}")

    print("\n=== 2. O'rgatish (to'liq batch) ===")
    print(f"  {'davr':>6} {'o_quv loss':>12} {'test loss':>11} "
          f"{'test aniqlik':>13}")
    for davr in range(1, 301):
        t.qadam(Xtr, ytr, lr=0.5)
        if davr in (1, 10, 50, 100, 200, 300):
            p_tr = t.bashorat(Xtr)
            p_te = t.bashorat(Xte)
            print(f"  {davr:>6} {log_loss(ytr, p_tr):>12.4f} "
                  f"{log_loss(yte, p_te):>11.4f} "
                  f"{accuracy_score(yte, p_te.argmax(1)):>13.4f}")

    print("\n=== 3. Mini-batch bilan ===")
    t2 = Tarmoq([20, 64, 32, 4], seed=0)
    rng = np.random.default_rng(0)
    print(f"  {'davr':>6} {'o_quv loss':>12} {'test aniqlik':>13}")
    for davr in range(1, 31):
        tartib = rng.permutation(len(ytr))
        for boshi in range(0, len(ytr), 64):
            idx = tartib[boshi:boshi + 64]
            t2.qadam(Xtr[idx], ytr[idx], lr=0.1)
        if davr in (1, 5, 10, 20, 30):
            p_te = t2.bashorat(Xte)
            print(f"  {davr:>6} {log_loss(ytr, t2.bashorat(Xtr)):>12.4f} "
                  f"{accuracy_score(yte, p_te.argmax(1)):>13.4f}")
    print("  mini-batch 30 davrda 300 davrlik natijaga yetdi")

    print("\n=== 4. Qatlamlar statistikasi (o'rgatilgandan keyin) ===")
    A_lar, Z_lar = t2.oldinga(Xte)
    print(f"  {'bosqich':<12} {'shakl':>12} {'std':>9} {'nol %':>8}")
    nomlar = ["kirish", "qatlam 1", "qatlam 2", "chiqish"]
    for nom, A in zip(nomlar, A_lar):
        print(f"  {nom:<12} {str(A.shape):>12} {A.std():>9.4f} "
              f"{(A == 0).mean():>8.1%}")

    print("\n=== 5. Og'irliklar qanday o'zgardi ===")
    yangi = Tarmoq([20, 64, 32, 4], seed=0)
    print(f"  {'qatlam':>7} {'boshlang_ich std':>18} "
          f"{'o_rgatilgan std':>17} {'nisbat':>9}")
    for i, (W0, W1) in enumerate(zip(yangi.Ws, t2.Ws)):
        print(f"  {i + 1:>7} {W0.std():>18.4f} {W1.std():>17.4f} "
              f"{W1.std() / W0.std():>9.3f}")
    print("  ⭐ 60 qatorlik backprop haqiqiy tarmoqni o'rgatdi")


if __name__ == "__main__":
    main()

=== 1. Tarmoq ===
  arxitektura: 20 -> 64 -> 32 -> 4
  parametrlar: 3556
  boshlang'ich test aniqlik: 0.2500

=== 2. O'rgatish (to'liq batch) ===
    davr   o_quv loss   test loss  test aniqlik
       1       1.8737      1.8786        0.2892
      10       0.9963      1.0465        0.6050
      50       0.6769      0.8550        0.7008
     100       0.5664      0.8349        0.7217
     200       0.3869      0.7941        0.7708
     300       0.3077      0.8387        0.7592

=== 3. Mini-batch bilan ===
    davr   o_quv loss  test aniqlik
       1       1.0085        0.5758
       5       0.6967        0.7050
      10       0.5575        0.7408
      20       0.3950        0.7733
      30       0.3450        0.7558
  mini-batch 30 davrda 300 davrlik natijaga yetdi

=== 4. Qatlamlar statistikasi (o'rgatilgandan keyin) ===
  bosqich             shakl       std    nol %
  kirish         (1200, 20)    0.9846     0.0%
  qatlam 1       (1200, 64)    0.8854    49.6%
  qatlam 2       (1200, 32)    1.0871    53.4%
  chiqish         (1200, 4)    2.8880     0.0%

=== 5. Og'irliklar qanday o'zgardi ===
   qatlam   boshlang_ich std   o_rgatilgan std    nisbat
        1             0.3105            0.3334     1.074
        2             0.1792            0.2099     1.171
        3             0.2647            0.4560     1.723
  ⭐ 60 qatorlik backprop haqiqiy tarmoqni o'rgatdi

dW = dZ.T @ A_oldingi                    # ⚠️ shakl teskari
dW = A_oldingi.T @ dZ                    # ✅

dZ = dA * (A > 0)                        # ⚠️ ReLU da tasodifan ishlaydi
dZ = dA * (Z > 0)                        # ✅ umumiy to'g'ri

dZ = P - Y                               # ⚠️ gradient N barobar katta
dZ = (P - Y) / N                         # ✅

db = dZ.mean(axis=0)                     # ⚠️ dZ allaqachon /N bo'lgan
db = dZ.sum(axis=0)                      # ✅

dZ = (P - Y) / N * P * (1 - P)           # ⚠️ ikki marta
dZ = (P - Y) / N                         # ✅

A_lar = eski_kesh                        # ⚠️ og'irlik o'zgargan
A_lar, Z_lar = self.oldinga(X)           # ✅ har qadamda qayta

# to'g'ridan-to'g'ri o'rgatishga o'tish   # ⚠️
# avval nisbiy xatoni tekshiring          # ✅

1.  # dZ formulasi?
2.  # dW formulasi?
3.  # db formulasi?
4.  # dA_oldingi formulasi?
5.  # dW shakli qanday bo'lishi kerak?
6.  # boshlang'ich gradient (softmax+CE)?
7.  # sonli hosila h qancha?
8.  # nisbiy xato chegarasi?
9.  # orqaga o'tish necha barobar qimmat?
10. # gradient clipping nima qiladi?
11. # f'(Z) yoki f'(A)?
12. # sonli hosila necha o'tish talab qiladi?

1.  dW = dZ.T @ A_oldingi

2.  dZ = dA * (A > 0)

3.  dZ = P - Y

4.  db = dZ.mean(axis=0)

5.  dZ = (P - Y) / N * P * (1 - P)