20.8-dars: Boshlang'ich qiymatlar va normalizatsiya

W = 0 BO'LSA:
  bir qatlamdagi HAMMA neyron bir xil chiqish beradi
  orqaga tarqalishda HAMMASIGA bir xil gradient keladi
  -> hammasi bir xil yangilanadi
  -> 64 neyronli qatlam AMALDA 1 neyron

BU "SIMMETRIYA MUAMMOSI"

W = DOIMIY (masalan hammasi 0.1) bo'lsa ham AYNI muammo

DEMAK: og'irliklar TASODIFIY bo'lishi SHART
       (bias ni nol qilish esa mumkin va odatiy)

E'TIBOR: bias ni nol qilish xavfsiz, chunki simmetriyani
         og'irliklardagi tasodifiylik allaqachon buzgan

Z = sum_{i=1}^{n} A_i * W_i     (n = n_kirish)

A va W mustaqil, o'rtachasi 0 bo'lsa:
  Var(Z) = n * Var(A) * Var(W)

TALAB: Var(Z) = Var(A)  (signal masshtabi saqlansin)
  =>  n * Var(W) = 1
  =>  Var(W) = 1 / n_kirish          <- XAVIER (tanh, sigmoid uchun)

RELU UCHUN TUZATISH:
  ReLU chiqishning ~yarmini nolga aylantiradi
  Var(ReLU(Z)) ~ 0.5 * Var(Z)
  =>  Var(W) = 2 / n_kirish          <- HE

ORQAGA O'TISH UCHUN: Var(W) = 1 / n_chiqish
XAVIER KOMPROMISSI: Var(W) = 2 / (n_kirish + n_chiqish)

NOMI              DISPERSIYA              QACHON
zeros             0                       HECH QACHON (og'irlik uchun)
normal(0, 0.01)   0.0001                  eski usul, chuqurda yomon
Xavier/Glorot     2/(n_in + n_out)        tanh, sigmoid
He/Kaiming        2/n_in                  ReLU va avlodlari
LeCun             1/n_in                  SELU
orthogonal        ortogonal matritsa      RNN
uniform variant   +-sqrt(3*Var)           normal o'rniga

BIAS: odatda 0
  istisno: ReLU da kichik musbat 0.01-bob - o'lishni kamaytiradi
  istisno: nomutanosib sinfda oxirgi bias = log(p/(1-p))

G'OYA: har qatlamda taqsimotni MAJBURAN normallashtirish

O'RGATISHDA (batch bo'yicha):
  mu = mean(Z, axis=0)
  var = var(Z, axis=0)
  Z_hat = (Z - mu) / sqrt(var + eps)
  chiqish = gamma * Z_hat + beta       <- gamma, beta O'RGANILADI

INFERENCE DA:
  o'rgatishda yig'ilgan HARAKATLANUVCHI o'rtacha va dispersiya

NIMA BERADI:
  boshlang'ich qiymatga sezgirlikni kamaytiradi
  kattaroq lr ishlatish imkonini beradi
  regularizatsiya ta'siri (batch shovqini)

KAMCHILIKLARI:
  batch kichik bo'lsa (< 16) statistika shovqinli
  ketma-ketliklarda noqulay
  o'rgatish/inference farqi xatolik manbai

BATCHNORM:  har BELGI bo'yicha, batch ichida
  mu shakli (d,)  - N ta namuna bo'yicha o'rtacha

LAYERNORM:  har NAMUNA bo'yicha, belgilar ichida
  mu shakli (N,)  - d ta belgi bo'yicha o'rtacha
  batch hajmiga BOG'LIQ EMAS
  transformerlarda standart (26-qism)

GROUPNORM:  belgilarni guruhlarga bo'lib
  CNN larda kichik batch bilan

RMSNORM:    LayerNorm ning soddalashtirilgani (markazlash yo'q)
  zamonaviy LLM larda (27-qism)

TANLASH:
  tabular/CNN + katta batch -> BatchNorm
  ketma-ketlik, transformer -> LayerNorm
  kichik batch             -> GroupNorm yoki LayerNorm

QISMAN HA:
  BatchNorm bilan noto'g'ri boshlanish TUZATILADI
  chunki har qatlamda taqsimot qayta markazlashtiriladi

LEKIN:
  birinchi qadamlar baribir yomon bo'ladi
  juda katta boshlanishda gradient portlashi qoladi
  normalizatsiyasiz qatlamlar (chiqish) himoyasiz qoladi

AMALIY QOIDA:
  to'g'ri boshlanish + normalizatsiya = eng yaxshi
  ikkalasidan bittasini tanlash kerak bo'lsa - BOSHLANISH
  (u bepul, normalizatsiya esa hisob qo'shadi)

import numpy as np

rng = np.random.default_rng(0)

# He (ReLU uchun)
W = rng.normal(0, np.sqrt(2.0 / n_in), (n_in, n_out))
# Xavier (tanh uchun)
W = rng.normal(0, np.sqrt(1.0 / n_in), (n_in, n_out))
# Xavier kompromissi
W = rng.normal(0, np.sqrt(2.0 / (n_in + n_out)), (n_in, n_out))
b = np.zeros(n_out)

# BatchNorm (o'rgatishda)
mu, var = Z.mean(axis=0), Z.var(axis=0)
Z_hat = (Z - mu) / np.sqrt(var + 1e-5)
chiqish = gamma * Z_hat + beta

# LayerNorm
mu = Z.mean(axis=1, keepdims=True)
var = Z.var(axis=1, keepdims=True)
chiqish = gamma * (Z - mu) / np.sqrt(var + 1e-5) + beta

nol og'irlik -> simmetriya -> o'rganmaydi
Var(W) = 1/n_in   Xavier (tanh)
Var(W) = 2/n_in   He (ReLU)
bias = 0
BatchNorm: batch bo'yicha · LayerNorm: namuna bo'yicha

"""Nol va doimiy boshlanish nima uchun ishlamaydi (real numpy)."""

import numpy as np


def softmax(Z):
    Z = Z - Z.max(axis=1, keepdims=True)
    E = np.exp(Z)
    return E / E.sum(axis=1, keepdims=True)


def orgat(W1, b1, W2, b2, X, y, qadamlar=200, lr=0.5):
    N = len(y)
    Y = np.zeros((N, W2.shape[1]))
    Y[np.arange(N), y] = 1.0
    for _ in range(qadamlar):
        Z1 = X @ W1 + b1
        A1 = np.maximum(0, Z1)
        Z2 = A1 @ W2 + b2
        dZ2 = (softmax(Z2) - Y) / N
        dW2 = A1.T @ dZ2
        db2 = dZ2.sum(axis=0)
        dZ1 = (dZ2 @ W2.T) * (Z1 > 0)
        dW1 = X.T @ dZ1
        db1 = dZ1.sum(axis=0)
        W1 -= lr * dW1
        b1 -= lr * db1
        W2 -= lr * dW2
        b2 -= lr * db2
    Z1 = X @ W1 + b1
    A1 = np.maximum(0, Z1)
    P = softmax(A1 @ W2 + b2)
    loss = -np.mean(np.log(np.clip(P[np.arange(N), y], 1e-15, None)))
    return loss, W1, A1


def main() -> None:
    rng = np.random.default_rng(0)
    N, d, H, K = 400, 10, 8, 3
    X = rng.normal(0, 1, (N, d))
    y = rng.integers(0, K, N)

    print("=== 1. Uch xil boshlanish ===")
    variantlar = {
        "nol": lambda: (np.zeros((d, H)), np.zeros((H, K))),
        "doimiy 0.1": lambda: (np.full((d, H), 0.1),
                               np.full((H, K), 0.1)),
        "tasodifiy He": lambda: (
            rng.normal(0, np.sqrt(2 / d), (d, H)),
            rng.normal(0, np.sqrt(2 / H), (H, K))),
    }
    print(f"  {'boshlanish':<14} {'yakuniy loss':>14} "
          f"{'noyob neyron':>14}")
    for nom, yasa in variantlar.items():
        W1, W2 = yasa()
        loss, W1_oxir, A1 = orgat(W1.copy(), np.zeros(H), W2.copy(),
                                  np.zeros(K), X, y)
        noyob = len({tuple(np.round(W1_oxir[:, j], 6))
                     for j in range(H)})
        print(f"  {nom:<14} {loss:>14.6f} {noyob:>14}")
    print(f"  (8 neyrondan nechtasi HAR XIL)")

    print("\n=== 2. Nol boshlanishda og'irliklar ===")
    W1, W2 = np.zeros((d, H)), np.zeros((H, K))
    _, W1_oxir, _ = orgat(W1, np.zeros(H), W2, np.zeros(K), X, y)
    print(f"  o'rgatishdan keyin W1 ning birinchi 3 ustuni:")
    for j in range(3):
        print(f"    neyron {j}: {W1_oxir[:4, j].round(6)}")
    print(f"  hamma ustun bir xilmi: "
          f"{bool(np.allclose(W1_oxir[:, 0:1], W1_oxir))}")

    print("\n=== 3. Tasodifiy boshlanishda ===")
    W1 = rng.normal(0, np.sqrt(2 / d), (d, H))
    W2 = rng.normal(0, np.sqrt(2 / H), (H, K))
    _, W1_oxir, A1 = orgat(W1, np.zeros(H), W2, np.zeros(K), X, y)
    print(f"  o'rgatishdan keyin W1 ning birinchi 3 ustuni:")
    for j in range(3):
        print(f"    neyron {j}: {W1_oxir[:4, j].round(4)}")
    print(f"  hamma ustun bir xilmi: "
          f"{bool(np.allclose(W1_oxir[:, 0:1], W1_oxir))}")

    print("\n=== 4. Bias ni nol qilish xavfsizmi ===")
    print(f"  {'holat':<26} {'yakuniy loss':>14}")
    for nom, b_yasa in [("bias = 0", lambda: np.zeros(H)),
                        ("bias = 0.01", lambda: np.full(H, 0.01)),
                        ("bias tasodifiy", lambda: rng.normal(0, 0.1, H))]:
        W1 = rng.normal(0, np.sqrt(2 / d), (d, H))
        W2 = rng.normal(0, np.sqrt(2 / H), (H, K))
        loss, _, _ = orgat(W1, b_yasa(), W2, np.zeros(K), X, y)
        print(f"  {nom:<26} {loss:>14.6f}")
    print("  og'irlik tasodifiy bo'lsa - bias ni nol qilish yetarli")

    print("\n=== 5. Neyronlar turlichaligini o'lchash ===")
    W1 = rng.normal(0, np.sqrt(2 / d), (d, H))
    W2 = rng.normal(0, np.sqrt(2 / H), (H, K))
    _, _, A1 = orgat(W1, np.zeros(H), W2, np.zeros(K), X, y)
    K_mat = np.corrcoef(A1.T)
    yuqori = K_mat[np.triu_indices(H, k=1)]
    print(f"  neyronlar chiqishi orasidagi korrelyatsiya:")
    print(f"    o'rtacha |r|: {np.nanmean(np.abs(yuqori)):.4f}")
    print(f"    maksimal |r|: {np.nanmax(np.abs(yuqori)):.4f}")
    print("  ⭐ Past korrelyatsiya = neyronlar HAR XIL narsa o'rgangan")


if __name__ == "__main__":
    main()

=== 1. Uch xil boshlanish ===
  boshlanish       yakuniy loss   noyob neyron
  nol                  1.089102              1
  doimiy 0.1           1.074863              1
  tasodifiy He         0.978602              8
  (8 neyrondan nechtasi HAR XIL)

=== 2. Nol boshlanishda og'irliklar ===
  o'rgatishdan keyin W1 ning birinchi 3 ustuni:
    neyron 0: [0. 0. 0. 0.]
    neyron 1: [0. 0. 0. 0.]
    neyron 2: [0. 0. 0. 0.]
  hamma ustun bir xilmi: True

=== 3. Tasodifiy boshlanishda ===
  o'rgatishdan keyin W1 ning birinchi 3 ustuni:
    neyron 0: [-0.0091  0.5417  0.059   0.0772]
    neyron 1: [-0.1055  1.0428 -0.4899 -0.0474]
    neyron 2: [ 0.3928 -0.7702  0.1833  0.359 ]
  hamma ustun bir xilmi: False

=== 4. Bias ni nol qilish xavfsizmi ===
  holat                        yakuniy loss
  bias = 0                         0.951138
  bias = 0.01                      0.969198
  bias tasodifiy                   0.958254
  og'irlik tasodifiy bo'lsa - bias ni nol qilish yetarli

=== 5. Neyronlar turlichaligini o'lchash ===
  neyronlar chiqishi orasidagi korrelyatsiya:
    o'rtacha |r|: 0.1781
    maksimal |r|: 0.4628
  ⭐ Past korrelyatsiya = neyronlar HAR XIL narsa o'rgangan

"""Var(Z) = n * Var(A) * Var(W) formulasini o'lchash (real numpy)."""

import numpy as np


def main() -> None:
    rng = np.random.default_rng(0)

    print("=== 1. Bitta qatlamda dispersiya ===")
    N = 20000
    print(f"  {'n_in':>6} {'Var(W)':>10} {'nazariy Var(Z)':>16} "
          f"{'o_lchangan':>12}")
    for n_in in [10, 50, 200, 1000]:
        A = rng.normal(0, 1, (N, n_in))
        var_w = 0.05
        W = rng.normal(0, np.sqrt(var_w), (n_in, 1))
        Z = A @ W
        print(f"  {n_in:>6} {var_w:>10.4f} {n_in * 1.0 * var_w:>16.4f} "
              f"{Z.var():>12.4f}")
    print("  Var(Z) = n_in * Var(A) * Var(W) - aniq bajariladi")

    print("\n=== 2. Xavier: Var(W) = 1/n_in ===")
    print(f"  {'n_in':>6} {'Var(W)':>12} {'Var(A)':>10} {'Var(Z)':>10}")
    for n_in in [10, 50, 200, 1000]:
        A = rng.normal(0, 1, (N, n_in))
        W = rng.normal(0, np.sqrt(1.0 / n_in), (n_in, 1))
        Z = A @ W
        print(f"  {n_in:>6} {1.0 / n_in:>12.6f} {A.var():>10.4f} "
              f"{Z.var():>10.4f}")
    print("  Var(Z) ~ 1 - signal masshtabi SAQLANADI")

    print("\n=== 3. ReLU dispersiyani yarmiga tushiradi ===")
    print(f"  {'Var(Z)':>10} {'Var(relu(Z))':>14} {'nisbat':>9}")
    for s in [0.5, 1.0, 2.0, 4.0]:
        Z = rng.normal(0, np.sqrt(s), 200000)
        A = np.maximum(0, Z)
        print(f"  {Z.var():>10.4f} {A.var():>14.4f} "
              f"{A.var() / Z.var():>9.4f}")
    print("  nisbat ~0.34 (nazariy (1 - 1/pi)/2 ~ 0.341)")

    print("\n=== 4. He tuzatishi ===")
    print(f"  {'boshlanish':<14} {'10 qatlamdan keyin Var(A)':>28}")
    for nom, koef in [("Xavier (1/n)", 1.0), ("He (2/n)", 2.0)]:
        A = rng.normal(0, 1, (2000, 128))
        for _ in range(10):
            W = rng.normal(0, np.sqrt(koef / A.shape[1]),
                           (A.shape[1], 128))
            A = np.maximum(0, A @ W)
        print(f"  {nom:<14} {A.var():>28.6e}")
    print("  He bilan dispersiya saqlanadi, Xavier bilan so'nadi")

    print("\n=== 5. tanh uchun teskari ===")
    print(f"  {'boshlanish':<14} {'10 qatlamdan keyin Var(A)':>28}")
    for nom, koef in [("Xavier (1/n)", 1.0), ("He (2/n)", 2.0)]:
        A = rng.normal(0, 1, (2000, 128))
        for _ in range(10):
            W = rng.normal(0, np.sqrt(koef / A.shape[1]),
                           (A.shape[1], 128))
            A = np.tanh(A @ W)
        print(f"  {nom:<14} {A.var():>28.6f}")
    print("  tanh da He TO'YINISHGA olib keladi")

    print("\n=== 6. Uniform va normal ===")
    print("  Uniform(-a, a) da Var = a^2/3, demak a = sqrt(3*Var)")
    print(f"  {'usul':<20} {'kerakli Var':>13} {'o_lchangan':>12}")
    n_in = 100
    kerak = 2.0 / n_in
    W_n = rng.normal(0, np.sqrt(kerak), (n_in, 200))
    a = np.sqrt(3 * kerak)
    W_u = rng.uniform(-a, a, (n_in, 200))
    print(f"  {'normal':<20} {kerak:>13.6f} {W_n.var():>12.6f}")
    print(f"  {'uniform':<20} {kerak:>13.6f} {W_u.var():>12.6f}")
    print("  ⭐ Muhimi taqsimot turi emas, DISPERSIYA")


if __name__ == "__main__":
    main()

=== 1. Bitta qatlamda dispersiya ===
    n_in     Var(W)   nazariy Var(Z)   o_lchangan
      10     0.0500           0.5000       0.8363
      50     0.0500           2.5000       2.3563
     200     0.0500          10.0000       8.1727
    1000     0.0500          50.0000      50.7203
  Var(Z) = n_in * Var(A) * Var(W) - aniq bajariladi

=== 2. Xavier: Var(W) = 1/n_in ===
    n_in       Var(W)     Var(A)     Var(Z)
      10     0.100000     1.0013     0.8258
      50     0.020000     0.9992     0.9227
     200     0.005000     1.0009     1.0780
    1000     0.001000     0.9999     1.0274
  Var(Z) ~ 1 - signal masshtabi SAQLANADI

=== 3. ReLU dispersiyani yarmiga tushiradi ===
      Var(Z)   Var(relu(Z))    nisbat
      0.5003         0.1720    0.3438
      0.9910         0.3382    0.3412
      2.0008         0.6873    0.3435
      3.9818         1.3579    0.3410
  nisbat ~0.34 (nazariy (1 - 1/pi)/2 ~ 0.341)

=== 4. He tuzatishi ===
  boshlanish        10 qatlamdan keyin Var(A)
  Xavier (1/n)                   1.468516e-03
  He (2/n)                       6.564601e-01
  He bilan dispersiya saqlanadi, Xavier bilan so'nadi

=== 5. tanh uchun teskari ===
  boshlanish        10 qatlamdan keyin Var(A)
  Xavier (1/n)                       0.053266
  He (2/n)                           0.309929
  tanh da He TO'YINISHGA olib keladi

=== 6. Uniform va normal ===
  Uniform(-a, a) da Var = a^2/3, demak a = sqrt(3*Var)
  usul                   kerakli Var   o_lchangan
  normal                    0.020000     0.020237
  uniform                   0.020000     0.019919
  ⭐ Muhimi taqsimot turi emas, DISPERSIYA

"""Ikki normalizatsiya: nima bo'yicha va qanday farq (real numpy)."""

import numpy as np


def batchnorm(Z, gamma=1.0, beta=0.0, eps=1e-5):
    mu = Z.mean(axis=0)
    var = Z.var(axis=0)
    return gamma * (Z - mu) / np.sqrt(var + eps) + beta


def layernorm(Z, gamma=1.0, beta=0.0, eps=1e-5):
    mu = Z.mean(axis=1, keepdims=True)
    var = Z.var(axis=1, keepdims=True)
    return gamma * (Z - mu) / np.sqrt(var + eps) + beta


def main() -> None:
    rng = np.random.default_rng(0)
    Z = rng.normal(5.0, 3.0, (6, 4)) * np.array([1.0, 0.1, 10.0, 2.0])

    print("=== 1. Xom Z ===")
    print(f"  shakl {Z.shape}, (namuna, belgi)")
    print(f"  {'belgi':>7} {'o_rtacha':>10} {'std':>9}")
    for j in range(4):
        print(f"  {j:>7} {Z[:, j].mean():>10.4f} {Z[:, j].std():>9.4f}")

    print("\n=== 2. BatchNorm dan keyin (ustunlar bo'yicha) ===")
    B = batchnorm(Z)
    print(f"  {'belgi':>7} {'o_rtacha':>10} {'std':>9}")
    for j in range(4):
        print(f"  {j:>7} {B[:, j].mean():>10.6f} {B[:, j].std():>9.6f}")
    print("  har BELGI ning o'rtachasi 0, std 1")

    print("\n=== 3. LayerNorm dan keyin (qatorlar bo'yicha) ===")
    L = layernorm(Z)
    print(f"  {'namuna':>7} {'o_rtacha':>10} {'std':>9}")
    for i in range(6):
        print(f"  {i:>7} {L[i].mean():>10.6f} {L[i].std():>9.6f}")
    print("  har NAMUNA ning o'rtachasi 0, std 1")

    print("\n=== 4. Batch hajmiga bog'liqlik ===")
    katta = rng.normal(0, 1, (512, 8))
    print(f"  {'batch':>7} {'BN std farqi':>15} {'LN std farqi':>15}")
    toliq_bn = batchnorm(katta)
    for b in [2, 4, 16, 64, 256]:
        kesim = katta[:b]
        bn = batchnorm(kesim)
        ln = layernorm(kesim)
        print(f"  {b:>7} {abs(bn.std() - toliq_bn.std()):>15.6f} "
              f"{abs(ln.std() - layernorm(katta).std()):>15.6f}")
    print("  LN batch hajmiga BOG'LIQ EMAS")

    print("\n=== 5. Chuqur tarmoqda ta'siri ===")
    print(f"  {'holat':<22} {'10 qatlamdan keyin std':>25}")
    for nom, norm in [("normalizatsiyasiz", None),
                      ("BatchNorm", batchnorm),
                      ("LayerNorm", layernorm)]:
        A = rng.normal(0, 1, (256, 64))
        for _ in range(10):
            W = rng.normal(0, 0.05, (64, 64))      # ataylab KICHIK
            Z2 = A @ W
            if norm is not None:
                Z2 = norm(Z2)
            A = np.maximum(0, Z2)
        print(f"  {nom:<22} {A.std():>25.6e}")
    print("  normalizatsiya yomon boshlanishni TUZATADI")

    print("\n=== 6. Inference rejimi ===")
    oquv = rng.normal(2.0, 1.5, (1000, 3))
    harakat_mu = oquv.mean(axis=0)
    harakat_var = oquv.var(axis=0)
    yangi = rng.normal(2.0, 1.5, (4, 3))
    xato_rejim = batchnorm(yangi)
    togri_rejim = (yangi - harakat_mu) / np.sqrt(harakat_var + 1e-5)
    print(f"  4 ta namunali batch:")
    print(f"  {'namuna':>7} {'xato (batch stat)':>20} "
          f"{'to_g_ri (harakat stat)':>24}")
    for i in range(4):
        print(f"  {i:>7} {str(xato_rejim[i].round(3)):>20} "
              f"{str(togri_rejim[i].round(3)):>24}")
    print("  ⭐ Inference da HARAKATLANUVCHI statistikani ishlating")


if __name__ == "__main__":
    main()

=== 1. Xom Z ===
  shakl (6, 4), (namuna, belgi)
    belgi   o_rtacha       std
        0     2.9443    2.3556
        1     0.4898    0.2374
        2    49.1084   26.4328
        3    11.7551    3.5857

=== 2. BatchNorm dan keyin (ustunlar bo'yicha) ===
    belgi   o_rtacha       std
        0   0.000000  0.999999
        1   0.000000  0.999911
        2   0.000000  1.000000
        3  -0.000000  1.000000
  har BELGI ning o'rtachasi 0, std 1

=== 3. LayerNorm dan keyin (qatorlar bo'yicha) ===
   namuna   o_rtacha       std
        0   0.000000  1.000000
        1  -0.000000  1.000000
        2   0.000000  1.000000
        3   0.000000  1.000000
        4  -0.000000  1.000000
        5  -0.000000  1.000000
  har NAMUNA ning o'rtachasi 0, std 1

=== 4. Batch hajmiga bog'liqlik ===
    batch    BN std farqi    LN std farqi
        2        0.000101        0.000005
        4        0.000010        0.000001
       16        0.000001        0.000001
       64        0.000000        0.000001
      256        0.000000        0.000000
  LN batch hajmiga BOG'LIQ EMAS

=== 5. Chuqur tarmoqda ta'siri ===
  holat                     10 qatlamdan keyin std
  normalizatsiyasiz                   1.658777e-06
  BatchNorm                           5.902359e-01
  LayerNorm                           6.328509e-01
  normalizatsiya yomon boshlanishni TUZATADI

=== 6. Inference rejimi ===
  4 ta namunali batch:
   namuna    xato (batch stat)   to_g_ri (harakat stat)
        0 [-1.625 -1.224  1.295]   [-0.957 -0.92   1.164]
        1 [ 1.051  1.198 -1.473]   [-0.231  0.876 -1.716]
        2 [ 0.5   -0.717 -0.173]   [-0.381 -0.544 -0.363]
        3  [0.074 0.743 0.351]   [-0.496  0.539  0.182]
  ⭐ Inference da HARAKATLANUVCHI statistikani ishlating

"""To'liq o'rgatishda boshlanish va BatchNorm ta'siri."""

import numpy as np
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler


def softmax(Z):
    Z = Z - Z.max(axis=1, keepdims=True)
    E = np.exp(Z)
    return E / E.sum(axis=1, keepdims=True)


class Tarmoq:
    def __init__(self, olchamlar, boshlanish="he", bn=False, seed=0):
        rng = np.random.default_rng(seed)
        self.bn = bn
        self.Ws, self.bs = [], []
        for a, b in zip(olchamlar[:-1], olchamlar[1:]):
            if boshlanish == "he":
                s = np.sqrt(2.0 / a)
            elif boshlanish == "xavier":
                s = np.sqrt(1.0 / a)
            elif boshlanish == "kichik":
                s = 0.01
            else:
                s = 0.5
            self.Ws.append(rng.normal(0, s, (a, b)))
            self.bs.append(np.zeros(b))

    def oldinga(self, X):
        A_lar, Z_lar = [X], []
        A = X
        for i, (W, b) in enumerate(zip(self.Ws, self.bs)):
            Z = A @ W + b
            if self.bn and i < len(self.Ws) - 1:
                Z = (Z - Z.mean(axis=0)) / np.sqrt(Z.var(axis=0) + 1e-5)
            Z_lar.append(Z)
            A = np.maximum(0, Z) if i < len(self.Ws) - 1 else Z
            A_lar.append(A)
        return A_lar, Z_lar

    def qadam(self, X, y, lr):
        N = len(y)
        A_lar, Z_lar = self.oldinga(X)
        Y = np.zeros_like(A_lar[-1])
        Y[np.arange(N), y] = 1.0
        dZ = (softmax(A_lar[-1]) - Y) / N
        for i in range(len(self.Ws) - 1, -1, -1):
            dW = A_lar[i].T @ dZ
            db = dZ.sum(axis=0)
            if i > 0:
                dZ = (dZ @ self.Ws[i].T) * (Z_lar[i - 1] > 0)
            self.Ws[i] -= lr * dW
            self.bs[i] -= lr * db

    def loss(self, X, y):
        P = softmax(self.oldinga(X)[0][-1])
        return float(-np.mean(np.log(
            np.clip(P[np.arange(len(y)), y], 1e-15, None))))

    def aniqlik(self, X, y):
        return accuracy_score(y, self.oldinga(X)[0][-1].argmax(1))


def main() -> None:
    X, y = make_classification(n_samples=3000, n_features=20,
                               n_informative=10, n_redundant=4,
                               n_classes=3, flip_y=0.08, class_sep=1.0,
                               random_state=0)
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3,
                                          random_state=0, stratify=y)
    sc = StandardScaler().fit(Xtr)
    Xtr, Xte = sc.transform(Xtr), sc.transform(Xte)
    olchamlar = [20] + [48] * 6 + [3]
    print(f"  arxitektura: 20 -> 48 x6 -> 3 (7 qatlam)")

    print("\n=== 1. Boshlang'ich signal oqimi ===")
    print(f"  {'boshlanish':<12} {'chiqish std':>13} {'boshl. loss':>13}")
    for b in ["kichik", "xavier", "he", "katta"]:
        t = Tarmoq(olchamlar, boshlanish=b)
        chiqish = t.oldinga(Xtr)[0][-1]
        print(f"  {b:<12} {chiqish.std():>13.4e} "
              f"{t.loss(Xtr, ytr):>13.4f}")
    print(f"  tasodifiy taxmin loss = {np.log(3):.4f}")

    print("\n=== 2. O'rgatish natijasi ===")
    print(f"  {'boshlanish':<12} {'BN':>5} {'20 davr':>9} "
          f"{'100 davr':>10} {'test':>8}")
    for b in ["kichik", "xavier", "he"]:
        for bn in [False, True]:
            t = Tarmoq(olchamlar, boshlanish=b, bn=bn)
            oraliq = None
            for davr in range(1, 101):
                t.qadam(Xtr, ytr, lr=0.2)
                if davr == 20:
                    oraliq = t.loss(Xtr, ytr)
            print(f"  {b:<12} {str(bn):>5} {oraliq:>9.4f} "
                  f"{t.loss(Xtr, ytr):>10.4f} "
                  f"{t.aniqlik(Xte, yte):>8.4f}")

    print("\n=== 3. Qatlamlar bo'ylab std (o'rgatishdan oldin) ===")
    print(f"  {'qatlam':>7}", end="")
    for b in ["kichik", "he", "katta"]:
        print(f" {b:>14}", end="")
    print()
    oqimlar = {b: Tarmoq(olchamlar, boshlanish=b).oldinga(Xtr)[0]
               for b in ["kichik", "he", "katta"]}
    for i in [1, 3, 5, 7]:
        print(f"  {i:>7}", end="")
        for b in ["kichik", "he", "katta"]:
            print(f" {oqimlar[b][i].std():>14.3e}", end="")
        print()

    print("\n=== 4. BatchNorm bilan oqim ===")
    print(f"  {'qatlam':>7}", end="")
    for b in ["kichik", "he", "katta"]:
        print(f" {b:>14}", end="")
    print()
    oqimlar_bn = {b: Tarmoq(olchamlar, boshlanish=b, bn=True)
                  .oldinga(Xtr)[0] for b in ["kichik", "he", "katta"]}
    for i in [1, 3, 5, 7]:
        print(f"  {i:>7}", end="")
        for b in ["kichik", "he", "katta"]:
            print(f" {oqimlar_bn[b][i].std():>14.3e}", end="")
        print()
    print("  BN bilan hamma boshlanish bir xil oqim beradi")

    print("\n=== 5. Xulosa jadvali ===")
    tavsiya = [
        ("ReLU oilasi", "He: sqrt(2/n_in)"),
        ("tanh, sigmoid", "Xavier: sqrt(1/n_in)"),
        ("SELU", "LeCun: sqrt(1/n_in)"),
        ("RNN", "orthogonal"),
        ("bias", "0 (ReLU da 0.01 ham mumkin)"),
    ]
    print(f"  {'holat':<16} {'boshlanish'}")
    for a, b in tavsiya:
        print(f"  {a:<16} {b}")
    print("  ⭐ To'g'ri boshlanish bepul, normalizatsiya narxi bor")


if __name__ == "__main__":
    main()

  arxitektura: 20 -> 48 x6 -> 3 (7 qatlam)

=== 1. Boshlang'ich signal oqimi ===
  boshlanish     chiqish std   boshl. loss
  kichik          3.5391e-10        1.0986
  xavier          7.1557e-02        1.1037
  he              8.0958e-01        1.4005
  katta           2.7649e+02       22.4183
  tasodifiy taxmin loss = 1.0986

=== 2. O'rgatish natijasi ===
  boshlanish      BN   20 davr   100 davr     test
  kichik       False    1.0984     1.0984   0.3411
  kichik        True    0.9361     0.8005   0.5844
  xavier       False    1.0465     0.4953   0.7922
  xavier        True    0.8942     0.5988   0.6611
  he           False    0.8518     0.4554   0.7522
  he            True    0.7874     0.5207   0.6911

=== 3. Qatlamlar bo'ylab std (o'rgatishdan oldin) ===
   qatlam         kichik             he          katta
        1      2.513e-02      7.947e-01      1.257e+00
        3      5.214e-05      6.870e-01      6.517e+00
        5      1.349e-07      7.407e-01      4.216e+01
        7      3.539e-10      8.096e-01      2.765e+02

=== 4. BatchNorm bilan oqim ===
   qatlam         kichik             he          katta
        1      5.830e-01      5.849e-01      5.849e-01
        3      5.850e-01      5.870e-01      5.870e-01
        5      5.974e-01      5.994e-01      5.994e-01
        7      4.901e-02      1.004e+00      2.460e+00
  BN bilan hamma boshlanish bir xil oqim beradi

=== 5. Xulosa jadvali ===
  holat            boshlanish
  ReLU oilasi      He: sqrt(2/n_in)
  tanh, sigmoid    Xavier: sqrt(1/n_in)
  SELU             LeCun: sqrt(1/n_in)
  RNN              orthogonal
  bias             0 (ReLU da 0.01 ham mumkin)
  ⭐ To'g'ri boshlanish bepul, normalizatsiya narxi bor

W = np.zeros((n_in, n_out))                  # ⚠️
W = rng.normal(0, np.sqrt(2 / n_in), ...)    # ✅

# ReLU uchun sqrt(1/n)                       # ⚠️ signal so'nadi
# ReLU uchun sqrt(2/n)                       # ✅

np.sqrt(2 / n_out)                           # ⚠️ oldinga o'tish buziladi
np.sqrt(2 / n_in)                            # ✅

batch = 4;  BatchNorm()                      # ⚠️ shovqinli statistika
batch = 4;  LayerNorm()                      # ✅

Z = (Z - Z.mean(0)) / Z.std(0)               # ⚠️ 1 namunada NaN
Z = (Z - harakat_mu) / np.sqrt(harakat_var)  # ✅

Z = X @ W + b;  Z = batchnorm(Z)             # ⚠️ b bekor bo'ladi
Z = X @ W;      Z = batchnorm(Z)             # ✅

# to'g'ridan-to'g'ri o'rgatish                # ⚠️
print([A.std() for A in oldinga(X)[0]])       # ✅

1.  # nol og'irlik nima qiladi?
2.  # bias ni nol qilish mumkinmi?
3.  # Var(Z) formulasi?
4.  # Xavier dispersiyasi?
5.  # He dispersiyasi?
6.  # nega 2/n?
7.  # BatchNorm qaysi o'q bo'yicha?
8.  # LayerNorm-chi?
9.  # kichik batchda qaysi biri?
10. # inference da BN nima ishlatadi?
11. # BN dan keyin bias kerakmi?
12. # boshlang'ich loss K sinfda?

1.  W = np.zeros((n_in, n_out))

2.  np.sqrt(2 / n_out)

3.  batch = 4;  BatchNorm()

4.  Z = (Z - Z.mean(0)) / Z.std(0)    # inference

5.  Z = X @ W + b;  Z = batchnorm(Z)