24.4-dars: Pozitsion kodlash

SELF-ATTENTION 24.2-bob:
  Q = X W_q,  K = X W_k,  V = X W_v
  chiqish = softmax(Q K^T / sqrt(d)) V

PERMUTATSIYA P (qatorlar joyini almashtirish):
  X' = P X  ->  Q' = P Q,  K' = P K,  V' = P V
  softmax(P Q K^T P^T) P V = P softmax(Q K^T) V
  chiqish' = P * chiqish      <- EKVIVARIANTLIK

MA'NOSI:
  har so'zning chiqish vektori tartibdan QAT'I NAZAR bir xil,
  faqat qator joyi almashadi
  o'rtacha pooling qo'shilsa: P butunlay yo'qoladi  <- INVARIANTLIK
  "ali vali dan oldin" va "vali ali dan oldin" - bir xil vektor

1-MISOL: max |f(PX) - P f(X)| = 2.4e-07  (son xatosi)
2-MISOL: pozitsiyasiz model tartib vazifasida 0.5053 (tasodif 0.5),
         sanoq vazifasida 1.0000

FORMULA (Vaswani va boshq., 2017):
  PE[p, 2i]   = sin(p * w_i)
  PE[p, 2i+1] = cos(p * w_i)
  w_i = 1 / 10000^(2i / d),   i = 0 .. d/2 - 1

  kirish: x_p = E[token_p] + PE[p]      (qo'shiladi, ulanmaydi)

XOSSALAR (1-misol, d = 16):
  qiymatlar [-1, 1] oralig'ida, norma DOIM sqrt(d/2) = 2.8284
  i = 0 -> to'lqin uzunligi 2*pi = 6.28 pozitsiya (tez o'zgaradi)
  i = 7 -> 19869 pozitsiya (deyarli o'zgarmaydi)
  "soat millari": sekundlik, minutlik, soatlik ... strelkalar

NIMA UCHUN KO'P CHASTOTA:
  bitta sinus davriy - p va p + 2*pi*n farqlanmaydi
  turli chastotalar birgalikda har pozitsiyaga yagona "barmoq izi"
  1-misol: 100 pozitsiyada 100 ta har xil vektor

BITTA CHASTOTA JUFTLIGI:
  [sin(w(p+k)), cos(w(p+k))] = R(w k) [sin(w p), cos(w p)]
  R(a) = [[ cos a,  sin a],
          [-sin a,  cos a]]           <- 2x2 burish matritsasi

BUTUN VEKTOR:
  PE[p + k] = M_k PE[p],   M_k = blok-diagonal (R(w_0 k), ..., R(w_{d/2-1} k))
  M_k faqat k ga bog'liq, p ga EMAS

1-MISOL:
  k = 1, 5, 20: max |PE[p+k] - M_k PE[p]| ~ 3e-15 (mashina aniqligi)
  M_5 ni faqat p < 50 dan topib (eng kichik kvadratlar), p >= 50 da:
    max xato 5.4e-07

SKALYAR KO'PAYTMA:
  PE[p] . PE[p+k] = yig'indi_i cos(w_i k)     <- faqat k ga bog'liq
  1-misol: k = +5 da p = 10, 30, 70 uchun hammasi 6.1370
  k = -5 da ham 6.1370: skalyar ko'paytma yo'nalishni bilmaydi

G'OYA:
  model "3 qadam oldingi so'z" ni topishi uchun
  mutlaq pozitsiyani emas, chiziqli almashtirishni o'rganishi mumkin

G'OYA: pozitsiyani ham token kabi embedding qilish
  pos = nn.Embedding(maks_uzunlik, d)
  x_p = E[token_p] + pos(p)
  BERT (512), GPT-2 (1024) - shu usul

AFZALLIGI:
  ma'lumotdan o'rganadi, formula tanlash shart emas
  qisqa ketma-ketlikda sinus bilan deyarli teng
  (2-misol: ikkalasi ham tartib vazifasida 1.0000)

KAMCHILIGI:
  parametr: maks_uzunlik * d  (512 * 768 = 393 216)
  maks_uzunlik dan uzun ketma-ketlik - jadvalda qator YO'Q (IndexError)
  jadval kattaroq bo'lsa ham - o'quvda ko'rilmagan qatorlar
  tasodifiy init holida qoladi (3-misol: p >= 16 da 0.140, tasodif 0.125)

SAVOL: L = 16 da o'rgatilgan model L = 64 da ishlaydimi?

3-MISOL (vazifa: har pozitsiyada 3 qadam oldingi tokenni aytish):
                L=16    L=32    L=64    p >= 16 (L=64)
  pozitsiyasiz  0.268   0.218   0.186   0.187
  sinus         1.000   0.561   0.391   0.235
  o'rganiladigan 1.000  0.530   0.322   0.140
  tasodif: 0.125

NIMA UCHUN SINUS HAM YIQILADI:
  formula PE[40] ni beradi, lekin model PE[40] + E[token] kombinatsiyasini
  hech qachon ko'rmagan; W_q, W_k shu hududda o'rgatilmagan
  "nazariy imkoniyat" != "o'rganilgan xatti-harakat"

KO'RILGAN POZITSIYALAR HAM ZARAR KO'RADI:
  L=64 ichidagi p < 16 - sinus 0.967, o'rganiladigan 0.993
  (L=16 da ikkalasi 1.000): ko'p "begona" kalitlar attention ni chalg'itadi

AMALIYOT:
  o'quvda kerakli uzunlikni ko'rsating (yoki uzunlik bo'yicha bosqichli o'rgatish)
  uzunroq kontekst kerak bo'lsa: nisbiy usullar (RoPE, ALiBi) +
  qisqa qo'shimcha o'rgatish

G'OYA (Su va boshq., 2021):
  pozitsiyani x ga QO'SHISH o'rniga, q va k ni pozitsiyaga mos BURISH
  q_m' = R(m) q_m,   k_n' = R(n) k_n
  R(p) - blok-diagonal, har (2i, 2i+1) juftlik p * w_i burchakka

ASOSIY XOSSA:
  q_m' . k_n' = q_m^T R(m)^T R(n) k_n = q_m^T R(n - m) k_n
  -> skor FAQAT nisbiy masofa (n - m) ga bog'liq
  -> sinusdan farqli: yo'nalish ham saqlanadi (R(3) != R(-3))

KOD (juftlik bo'yicha):
  y[2i]   = x[2i] cos(p w_i) - x[2i+1] sin(p w_i)
  y[2i+1] = x[2i] sin(p w_i) + x[2i+1] cos(p w_i)

4-MISOL:
  norma o'zgarmaydi (3.3953 hamma p da)
  m - n = 3 bo'lgan (5, 2), (25, 22), (105, 102): skor 1.1783, oraliq 7.2e-07
  qo'shiladigan sinusda xuddi shu juftliklar: oraliq 5.3169
  ekstrapolyatsiya, p >= 16 (L=64): RoPE 0.510, sinus 0.235
    farq +0.2747, SE 0.0629 - sezilarli, lekin 1.000 dan uzoq

QAYERDA: LLaMA, Mistral, Qwen va boshqa ko'p zamonaviy til modellari
  parametr yo'q; faqat attention ichida q va k ga qo'llanadi (v ga emas)

import math
import torch
import torch.nn as nn

# sinusoidal (bufer - o'rganilmaydi, lekin .to(device) bilan ko'chadi)
def sinus_pe(n, d):
    p = torch.arange(n)[:, None].float()
    w = torch.exp(-math.log(10000.0) * torch.arange(0, d, 2).float() / d)
    pe = torch.zeros(n, d)
    pe[:, 0::2] = torch.sin(p * w)
    pe[:, 1::2] = torch.cos(p * w)
    return pe

self.register_buffer("pe", sinus_pe(maks, d))
x = self.emb(X) + self.pe[:X.shape[1]]            # (B, L, d) + (L, d)

# o'rganiladigan
self.pos = nn.Embedding(maks, d)
x = self.emb(X) + self.pos(torch.arange(X.shape[1]))

# RoPE: q, k (B, h, L, d_h) ga, attention dan OLDIN
def rope(x, p):
    a = p[:, None].float() * chastotalar(x.shape[-1])     # (L, d_h/2)
    c, s = torch.cos(a), torch.sin(a)
    x1, x2 = x[..., 0::2], x[..., 1::2]
    y = torch.empty_like(x)
    y[..., 0::2] = x1 * c - x2 * s
    y[..., 1::2] = x1 * s + x2 * c
    return y

q, k = rope(q, torch.arange(L)), rope(k, torch.arange(L))

# permutatsiya tekshiruvi
perm = torch.randperm(L)
(model(X)[:, perm] - model(X[:, perm])).abs().max()    # pozitsiyasiz ~ 1e-7

self-attention tartibni ko'rmaydi - pozitsiya alohida kerak
sinus: formula, parametrsiz, siljish = burish
o'rganiladigan: jadval, faqat ko'rilgan pozitsiyalar
ekstrapolyatsiya: formula != kafolat; o'lchang
RoPE: q va k ni burish, skor = f(n - m)

"""Self-attention permutatsiyaga befarq; sinusoidal kodlash noldan va uning siljish xossasi."""

import math

import numpy as np
import torch


def self_attention(X, Wq, Wk, Wv):
    q, k, v = X @ Wq, X @ Wk, X @ Wv
    w = torch.softmax(q @ k.T / math.sqrt(q.shape[1]), dim=1)
    return w @ v


def sinus_pe(L, d):
    """PE[p, 2i] = sin(p / 10000^(2i/d)),  PE[p, 2i+1] = cos(p / 10000^(2i/d))."""
    p = np.arange(L)[:, None]
    w = 1.0 / 10000 ** (np.arange(0, d, 2) / d)          # chastotalar, (d/2,)
    pe = np.zeros((L, d))
    pe[:, 0::2] = np.sin(p * w)
    pe[:, 1::2] = np.cos(p * w)
    return pe, w


def siljish_matritsasi(w, k):
    """PE[p + k] = M_k @ PE[p]: har chastota juftligi uchun 2x2 burish."""
    M = np.zeros((2 * len(w), 2 * len(w)))
    for i, wi in enumerate(w):
        c, s = np.cos(wi * k), np.sin(wi * k)
        M[2 * i:2 * i + 2, 2 * i:2 * i + 2] = [[c, s], [-s, c]]
    return M


def main() -> None:
    torch.manual_seed(0)
    d = 16
    lugat = ["men", "bugun", "kitob", "oldim"]
    E = {s: torch.randn(d) for s in lugat}
    Wq, Wk, Wv = (torch.randn(d, d) / math.sqrt(d) for _ in range(3))

    print("=== 1. Pozitsiyasiz self-attention ===")
    jumla = ["men", "bugun", "kitob", "oldim"]
    tartib = [2, 0, 3, 1]
    aralash = [jumla[i] for i in tartib]
    X = torch.stack([E[s] for s in jumla])
    Xp = torch.stack([E[s] for s in aralash])
    Y, Yp = self_attention(X, Wq, Wk, Wv), self_attention(Xp, Wq, Wk, Wv)
    print(f"  A: {' '.join(jumla)}")
    print(f"  B: {' '.join(aralash)}")
    farq = (Yp - Y[tartib]).abs().max().item()
    print(f"  max |f(B) - f(A) ning aralashtirilgani|: {farq:.1e}")
    ort = (Yp.mean(0) - Y.mean(0)).abs().max().item()
    print(f"  o'rtacha pooling farqi:                   {ort:.1e}")
    if farq < 1e-5:
        print("  har so'z vektori tartibdan qat'i nazar BIR XIL - faqat joyi almashdi")
    if ort < 1e-5:
        print("  pooling dan keyin A va B ni ajratib bo'lmaydi")

    print("\n=== 2. Sinusoidal kodlash (d = 16) ===")
    L = 100
    pe, w = sinus_pe(L, d)
    print(f"  {'p':>3} " + " ".join(f"{f'dim{j}':>7}" for j in (0, 1, 2, 3, 14, 15)))
    for p in (0, 1, 2, 3, 50):
        print(f"  {p:>3} " + " ".join(f"{pe[p, j]:>7.3f}" for j in (0, 1, 2, 3, 14, 15)))
    normalar = np.linalg.norm(pe, axis=1)
    print(f"  har pozitsiya normasi: min {normalar.min():.4f}, max {normalar.max():.4f}"
          f"  (sqrt(d/2) = {math.sqrt(d / 2):.4f})")
    tol = 2 * np.pi / w
    print(f"  to'lqin uzunliklari: {tol[0]:.2f} ... {tol[-1]:.0f} pozitsiya")
    takror = len({tuple(np.round(r, 6)) for r in pe})
    print(f"  {L} pozitsiyada har xil vektorlar soni: {takror}")

    print("\n=== 3. Nisbiy siljish - chiziqli almashtirish ===")
    for k in (1, 5, 20):
        M = siljish_matritsasi(w, k)
        xato = np.abs(pe[k:] - pe[:-k] @ M.T).max()
        print(f"  k = {k:>2}: max |PE[p+k] - M_k PE[p]| hamma p uchun = {xato:.1e}")
    k = 5
    A, B = pe[:50 - k], pe[k:50]
    M_fit, *_ = np.linalg.lstsq(A, B, rcond=None)
    xato_yangi = np.abs(pe[50:L - k] @ M_fit - pe[50 + k:]).max()
    print(f"  M_5 ni FAQAT p < 50 dan topib, p >= 50 da qo'llash: max xato {xato_yangi:.1e}")
    print("  M_k faqat k ga bog'liq, p ga emas: 'k qadam o'ngga' - bitta matritsa")

    print("\n=== 4. Skalyar ko'paytma faqat masofaga bog'liq ===")
    print(f"  {'':>10}" + "".join(f"{f'p={p}':>9}" for p in (10, 30, 70)))
    for k in (1, 5, 20, -5):
        qator = [pe[p] @ pe[p + k] for p in (10, 30, 70)]
        print(f"  {f'k = {k:+d}':>10}" + "".join(f"{v:>9.4f}" for v in qator))
    k5, km5 = pe[30] @ pe[35], pe[30] @ pe[25]
    if abs(k5 - km5) < 1e-9:
        print("  +5 va -5 bir xil: skalyar ko'paytma YO'NALISHNI bilmaydi")
    print("  ⭐ Sinusoidal kodlash: siljish = burish, o'xshashlik = masofa funksiyasi")


if __name__ == "__main__":
    main()

=== 1. Pozitsiyasiz self-attention ===
  A: men bugun kitob oldim
  B: kitob men oldim bugun
  max |f(B) - f(A) ning aralashtirilgani|: 2.4e-07
  o'rtacha pooling farqi:                   6.0e-08
  har so'z vektori tartibdan qat'i nazar BIR XIL - faqat joyi almashdi
  pooling dan keyin A va B ni ajratib bo'lmaydi

=== 2. Sinusoidal kodlash (d = 16) ===
    p    dim0    dim1    dim2    dim3   dim14   dim15
    0   0.000   1.000   0.000   1.000   0.000   1.000
    1   0.841   0.540   0.311   0.950   0.000   1.000
    2   0.909  -0.416   0.591   0.807   0.001   1.000
    3   0.141  -0.990   0.813   0.583   0.001   1.000
   50  -0.262   0.965  -0.103  -0.995   0.016   1.000
  har pozitsiya normasi: min 2.8284, max 2.8284  (sqrt(d/2) = 2.8284)
  to'lqin uzunliklari: 6.28 ... 19869 pozitsiya
  100 pozitsiyada har xil vektorlar soni: 100

=== 3. Nisbiy siljish - chiziqli almashtirish ===
  k =  1: max |PE[p+k] - M_k PE[p]| hamma p uchun = 2.7e-15
  k =  5: max |PE[p+k] - M_k PE[p]| hamma p uchun = 2.9e-15
  k = 20: max |PE[p+k] - M_k PE[p]| hamma p uchun = 2.7e-15
  M_5 ni FAQAT p < 50 dan topib, p >= 50 da qo'llash: max xato 5.4e-07
  M_k faqat k ga bog'liq, p ga emas: 'k qadam o'ngga' - bitta matritsa

=== 4. Skalyar ko'paytma faqat masofaga bog'liq ===
                 p=10     p=30     p=70
      k = +1   7.4852   7.4852   7.4852
      k = +5   6.1370   6.1370   6.1370
     k = +20   5.7755   5.7755   5.7755
      k = -5   6.1370   6.1370   6.1370
  +5 va -5 bir xil: skalyar ko'paytma YO'NALISHNI bilmaydi
  ⭐ Sinusoidal kodlash: siljish = burish, o'xshashlik = masofa funksiyasi

"""Pozitsiyasiz model tartibga bog'liq vazifani yecha olmaydi: sintetik tajriba."""

import math

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

SOZLAR = ["<pad>", "ali", "vali", "bugun", "maktabga", "keldi", "kitob",
          "oldi", "tez", "uyga", "ertalab"]
V, L = len(SOZLAR), 10
ALI, VALI = 1, 2


def sinus_pe(n, d):
    p = torch.arange(n)[:, None].float()
    w = torch.exp(-math.log(10000.0) * torch.arange(0, d, 2).float() / d)
    pe = torch.zeros(n, d)
    pe[:, 0::2] = torch.sin(p * w)
    pe[:, 1::2] = torch.cos(p * w)
    return pe


class Model(nn.Module):
    """Bitta self-attention bloki + o'rtacha pooling; pe: yoq, sinus, organ."""

    def __init__(self, pe, d=32, h=2):
        super().__init__()
        self.pe = pe
        self.emb = nn.Embedding(V, d)
        if pe == "organ":
            self.pos = nn.Embedding(L, d)
        self.register_buffer("sin", sinus_pe(L, d))
        self.att = nn.MultiheadAttention(d, h, batch_first=True)
        self.ln1, self.ln2 = nn.LayerNorm(d), nn.LayerNorm(d)
        self.ffn = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
        self.chiq = nn.Linear(d, 2)

    def forward(self, X):
        x = self.emb(X)
        if self.pe == "sinus":
            x = x + self.sin[:X.shape[1]]
        elif self.pe == "organ":
            x = x + self.pos(torch.arange(X.shape[1]))
        y = self.ln1(x)
        x = x + self.att(y, y, y, need_weights=False)[0]
        x = x + self.ffn(self.ln2(x))
        return self.chiq(x.mean(1))


def tartib_vazifa(n, g):
    """ali va vali bittadan; y = 1 agar ali vali dan OLDIN kelsa."""
    X = torch.randint(3, V, (n, L), generator=g)
    y = torch.randint(0, 2, (n,), generator=g)
    joy = torch.rand(n, L, generator=g).argsort(1)[:, :2].sort(1).values
    r = torch.arange(n)
    X[r, joy[:, 0]] = torch.where(y == 1, ALI, VALI)
    X[r, joy[:, 1]] = torch.where(y == 1, VALI, ALI)
    return X, y


def sanoq_vazifa(n, g):
    """ali va vali 0-3 martadan (teng emas); y = 1 agar ali KO'P bo'lsa."""
    X = torch.randint(3, V, (n, L), generator=g)
    y = torch.randint(0, 2, (n,), generator=g)
    juft = torch.rand(n, 4, generator=g).argsort(1)[:, :2]
    ko, oz = juft.max(1).values, juft.min(1).values
    ka = torch.where(y == 1, ko, oz)
    kv = torch.where(y == 1, oz, ko)
    rang = torch.rand(n, L, generator=g).argsort(1).argsort(1)
    X = torch.where(rang < ka[:, None], ALI, X)
    X = torch.where((rang >= ka[:, None]) & (rang < (ka + kv)[:, None]), VALI, X)
    return X, y


def orgat(pe, vazifa, seed, qadam=200):
    torch.manual_seed(seed)
    m = Model(pe)
    opt = torch.optim.Adam(m.parameters(), lr=0.003)
    g = torch.Generator().manual_seed(seed)
    for _ in range(qadam):
        X, y = vazifa(64, g)
        loss = F.cross_entropy(m(X), y)
        opt.zero_grad()
        loss.backward()
        opt.step()
    m.eval()
    return m


def korsat(X):
    return " ".join(SOZLAR[i] for i in X.tolist())


def main() -> None:
    torch.set_num_threads(1)
    print("=== 1. Ikki vazifa ===")
    X, y = tartib_vazifa(2, torch.Generator().manual_seed(3))
    for b in range(2):
        print(f"  TARTIB y={int(y[b])}: {korsat(X[b])}")
    X, y = sanoq_vazifa(2, torch.Generator().manual_seed(3))
    for b in range(2):
        print(f"  SANOQ  y={int(y[b])}: {korsat(X[b])}")
    print("  TARTIB: ali vali dan oldinmi?  SANOQ: ali ko'pmi?")

    testlar = {"tartib": tartib_vazifa(2000, torch.Generator().manual_seed(99)),
               "sanoq": sanoq_vazifa(2000, torch.Generator().manual_seed(98))}
    turlar = ["yoq", "sinus", "organ"]
    natija = {(v, pe): [] for v in testlar for pe in turlar}
    modellar = {}
    for s in range(3):
        for nom, vazifa in [("tartib", tartib_vazifa), ("sanoq", sanoq_vazifa)]:
            for pe in turlar:
                m = orgat(pe, vazifa, s)
                Xt, yt = testlar[nom]
                with torch.no_grad():
                    a = (m(Xt).argmax(1) == yt).float().mean().item()
                natija[(nom, pe)].append(a)
                modellar[(nom, pe, s)] = m

    print("\n=== 2. Test aniqligi (200 qadam, 3 seed o'rtachasi) ===")
    print(f"  {'vazifa':<8}" + "".join(f"{pe:>9}" for pe in turlar))
    for nom in testlar:
        print(f"  {nom:<8}" + "".join(f"{np.mean(natija[(nom, pe)]):>9.4f}"
                                      for pe in turlar))
    print("  tasodif darajasi: 0.5")

    print("\n=== 3. Juftlashgan farq: pozitsiyali - pozitsiyasiz ===")
    print(f"  {'vazifa':<8} {'pe':<6} {'farq':>8} {'SE':>7} {'sezilarli':>10}")
    for nom in testlar:
        a0 = np.array(natija[(nom, "yoq")])
        for pe in ["sinus", "organ"]:
            f = np.array(natija[(nom, pe)]) - a0
            se = f.std(ddof=1) / np.sqrt(len(f))
            print(f"  {nom:<8} {pe:<6} {f.mean():>+8.4f} {se:>7.4f} "
                  f"{str(abs(f.mean()) > 2 * se):>10}")

    print("\n=== 4. Nima uchun: pozitsiyasiz model tartibni KO'RMAYDI ===")
    m = modellar[("tartib", "yoq", 0)]
    Xt, yt = testlar["tartib"]
    perm = torch.randperm(L, generator=torch.Generator().manual_seed(1))
    with torch.no_grad():
        d = (m(Xt) - m(Xt[:, perm])).abs().max().item()
        teskari = torch.where(Xt == ALI, VALI, torch.where(Xt == VALI, ALI, Xt))
        almash = (m(Xt) - m(teskari)).abs().max().item()
    print(f"  max |logit(X) - logit(X aralashtirilgan)|:        {d:.1e}")
    print(f"  max |logit(X) - logit(ali va vali almashgan)|:    {almash:.1e}")
    print("  almashtirish yorliqni TESKARI qiladi, so'zlar to'plami esa o'sha")
    print("  -> ikkala yorliq uchun BIR XIL chiqish; eng yaxshi imkoniyat 0.5")
    m2 = modellar[("tartib", "sinus", 0)]
    with torch.no_grad():
        d2 = (m2(Xt) - m2(Xt[:, perm])).abs().max().item()
    print(f"  sinus bilan xuddi shu farq: {d2:.2f} - tartib endi signal")
    print("  ⭐ Pozitsiya ma'lumoti bo'lmasa, Transformer 'so'zlar qopi' modeli")


if __name__ == "__main__":
    main()

=== 1. Ikki vazifa ===
  TARTIB y=0: keldi vali maktabga kitob bugun bugun bugun ali tez kitob
  TARTIB y=1: keldi ali maktabga maktabga keldi vali ertalab uyga bugun oldi
  SANOQ  y=0: keldi bugun maktabga vali vali bugun bugun ali tez kitob
  SANOQ  y=1: keldi vali maktabga ali keldi ali ali vali bugun oldi
  TARTIB: ali vali dan oldinmi?  SANOQ: ali ko'pmi?

=== 2. Test aniqligi (200 qadam, 3 seed o'rtachasi) ===
  vazifa        yoq    sinus    organ
  tartib     0.5053   1.0000   1.0000
  sanoq      1.0000   1.0000   1.0000
  tasodif darajasi: 0.5

=== 3. Juftlashgan farq: pozitsiyali - pozitsiyasiz ===
  vazifa   pe         farq      SE  sezilarli
  tartib   sinus   +0.4947  0.0025       True
  tartib   organ   +0.4947  0.0025       True
  sanoq    sinus   +0.0000  0.0000      False
  sanoq    organ   +0.0000  0.0000      False

=== 4. Nima uchun: pozitsiyasiz model tartibni KO'RMAYDI ===
  max |logit(X) - logit(X aralashtirilgan)|:        8.9e-08
  max |logit(X) - logit(ali va vali almashgan)|:    8.9e-08
  almashtirish yorliqni TESKARI qiladi, so'zlar to'plami esa o'sha
  -> ikkala yorliq uchun BIR XIL chiqish; eng yaxshi imkoniyat 0.5
  sinus bilan xuddi shu farq: 27.69 - tartib endi signal
  ⭐ Pozitsiya ma'lumoti bo'lmasa, Transformer 'so'zlar qopi' modeli

"""Uzunlikdan tashqariga: 16 da o'rgatib, 32 va 64 da sinash (sinus va o'rganiladigan)."""

import math

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

V, SILJISH, L_OQUV, MAKS = 8, 3, 16, 64


def sinus_pe(n, d):
    p = torch.arange(n)[:, None].float()
    w = torch.exp(-math.log(10000.0) * torch.arange(0, d, 2).float() / d)
    pe = torch.zeros(n, d)
    pe[:, 0::2] = torch.sin(p * w)
    pe[:, 1::2] = torch.cos(p * w)
    return pe


class Model(nn.Module):
    """Pre-LN blok, har pozitsiyada 3 qadam oldingi tokenni topish."""

    def __init__(self, pe, d=32, h=2):
        super().__init__()
        self.pe, self.h = pe, h
        self.emb = nn.Embedding(V, d)
        if pe == "organ":
            self.pos = nn.Embedding(MAKS, d)      # 64 qator, o'quvda faqat 16 tasi
        self.register_buffer("sin", sinus_pe(MAKS, d))
        self.qkv, self.o = nn.Linear(d, 3 * d), nn.Linear(d, d)
        self.ln1, self.ln2, self.ln3 = nn.LayerNorm(d), nn.LayerNorm(d), nn.LayerNorm(d)
        self.ffn = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
        self.chiq = nn.Linear(d, V)

    def forward(self, X):
        B, L = X.shape
        x = self.emb(X)
        if self.pe == "sinus":
            x = x + self.sin[:L]
        elif self.pe == "organ":
            x = x + self.pos(torch.arange(L))
        q, k, v = (self.qkv(self.ln1(x)).reshape(B, L, 3, self.h, -1)
                   .permute(2, 0, 3, 1, 4))
        a = F.scaled_dot_product_attention(q, k, v)
        x = x + self.o(a.transpose(1, 2).reshape(B, L, -1))
        x = x + self.ffn(self.ln2(x))
        return self.chiq(self.ln3(x))


def batch(n, L, g):
    X = torch.randint(0, V, (n, L), generator=g)
    Y = torch.full((n, L), -100)
    Y[:, SILJISH:] = X[:, :-SILJISH]          # nishon: 3 pozitsiya oldingi token
    return X, Y


def orgat(pe, seed, qadam=400):
    torch.manual_seed(seed)
    m = Model(pe)
    opt = torch.optim.Adam(m.parameters(), lr=0.003)
    g = torch.Generator().manual_seed(seed)
    for _ in range(qadam):
        X, Y = batch(64, L_OQUV, g)
        loss = F.cross_entropy(m(X).reshape(-1, V), Y.reshape(-1))
        opt.zero_grad()
        loss.backward()
        opt.step()
    m.eval()
    return m


def olcha(m, L):
    X, Y = batch(500, L, torch.Generator().manual_seed(100 + L))
    with torch.no_grad():
        togri = m(X).argmax(-1) == Y
    ichki = togri[:, SILJISH:L_OQUV].float().mean().item()
    tashqi = togri[:, L_OQUV:].float().mean().item() if L > L_OQUV else float("nan")
    return togri[:, SILJISH:].float().mean().item(), ichki, tashqi


def main() -> None:
    torch.set_num_threads(1)
    print("=== 1. Vazifa ===")
    X, Y = batch(1, 10, torch.Generator().manual_seed(0))
    print(f"  kirish: {X[0].tolist()}")
    print(f"  nishon: {['-' if t < 0 else t for t in Y[0].tolist()]}")
    print(f"  o'rgatish uzunligi {L_OQUV}; test {L_OQUV}, 32, 64; tasodif 1/{V} = {1 / V:.3f}")

    turlar = ["yoq", "sinus", "organ"]
    natija = {pe: [] for pe in turlar}
    for s in range(3):
        for pe in turlar:
            m = orgat(pe, s)
            natija[pe].append([olcha(m, L) for L in (16, 32, 64)])
    r = {pe: np.array(v) for pe, v in natija.items()}     # (seed, uzunlik, 3)

    print("\n=== 2. Aniqlik uzunlik bo'yicha (400 qadam, 3 seed o'rtachasi) ===")
    print(f"  {'pe':<7} {'L=16':>7} {'L=32':>7} {'L=64':>7}")
    for pe in turlar:
        print(f"  {pe:<7}" + "".join(f"{r[pe][:, j, 0].mean():>8.3f}" for j in range(3)))

    print("\n=== 3. L=64 ichida: o'quvda ko'rilgan va ko'rilmagan pozitsiyalar ===")
    print(f"  {'pe':<7} {'p < 16':>8} {'p >= 16':>8}")
    for pe in turlar:
        print(f"  {pe:<7} {r[pe][:, 2, 1].mean():>8.3f} {r[pe][:, 2, 2].mean():>8.3f}")
    print(f"  (L=16 dagi p < 16: sinus {r['sinus'][:, 0, 1].mean():.3f}, "
          f"organ {r['organ'][:, 0, 1].mean():.3f})")

    print("\n=== 4. Juftlashgan farq, p >= 16 (L=64): sinus - organ ===")
    f = r["sinus"][:, 2, 2] - r["organ"][:, 2, 2]
    se = f.std(ddof=1) / np.sqrt(len(f))
    print(f"  seedlar bo'yicha farq: {np.round(f, 3).tolist()}")
    print(f"  o'rtacha {f.mean():+.4f}, SE {se:.4f}")
    if abs(f.mean()) <= 2 * se:
        print("  farq 2*SE ichida - ekstrapolyatsiyada ikkalasi teng")
    elif f.mean() > 0:
        print("  sinus sezilarli yaxshi - lekin mutlaq qiymatga qarang")
    else:
        print("  organ sezilarli yaxshi")
    eng = max(r[pe][:, 2, 2].mean() for pe in ["sinus", "organ"])
    if eng < 0.5:
        print(f"  ikkalasi ham ko'rilmagan pozitsiyalarda <= {eng:.3f}: "
              "ekstrapolyatsiya YO'Q")
    print("  ⭐ Formula har pozitsiya uchun vektor beradi, lekin model ularni ko'rmagan")


if __name__ == "__main__":
    main()

=== 1. Vazifa ===
  kirish: [4, 7, 5, 0, 3, 3, 3, 7, 1, 3]
  nishon: ['-', '-', '-', 4, 7, 5, 0, 3, 3, 3]
  o'rgatish uzunligi 16; test 16, 32, 64; tasodif 1/8 = 0.125

=== 2. Aniqlik uzunlik bo'yicha (400 qadam, 3 seed o'rtachasi) ===
  pe         L=16    L=32    L=64
  yoq       0.268   0.218   0.186
  sinus     1.000   0.561   0.391
  organ     1.000   0.530   0.322

=== 3. L=64 ichida: o'quvda ko'rilgan va ko'rilmagan pozitsiyalar ===
  pe        p < 16  p >= 16
  yoq        0.183    0.187
  sinus      0.967    0.235
  organ      0.993    0.140
  (L=16 dagi p < 16: sinus 1.000, organ 1.000)

=== 4. Juftlashgan farq, p >= 16 (L=64): sinus - organ ===
  seedlar bo'yicha farq: [0.131, 0.083, 0.071]
  o'rtacha +0.0948, SE 0.0183
  sinus sezilarli yaxshi - lekin mutlaq qiymatga qarang
  ikkalasi ham ko'rilmagan pozitsiyalarda <= 0.235: ekstrapolyatsiya YO'Q
  ⭐ Formula har pozitsiya uchun vektor beradi, lekin model ularni ko'rmagan

"""RoPE noldan: burish, nisbiy pozitsiya va uzunlikdan tashqariga sinov."""

import math

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

V, SILJISH, L_OQUV, MAKS = 8, 3, 16, 64


def chastotalar(d):
    return torch.exp(-math.log(10000.0) * torch.arange(0, d, 2).float() / d)


def sinus_pe(n, d):
    p = torch.arange(n)[:, None].float()
    pe = torch.zeros(n, d)
    pe[:, 0::2] = torch.sin(p * chastotalar(d))
    pe[:, 1::2] = torch.cos(p * chastotalar(d))
    return pe


def rope(x, pozitsiya):
    """x: (..., L, d). Har (2i, 2i+1) juftlik p * w_i burchakka buriladi."""
    burchak = pozitsiya[:, None].float() * chastotalar(x.shape[-1])
    c, s = torch.cos(burchak), torch.sin(burchak)
    x1, x2 = x[..., 0::2], x[..., 1::2]
    y = torch.empty_like(x)
    y[..., 0::2] = x1 * c - x2 * s
    y[..., 1::2] = x1 * s + x2 * c
    return y


class Model(nn.Module):
    """3-misoldagi model; pe = 'sinus' (kirishga qo'shiladi) yoki 'rope' (q, k ga)."""

    def __init__(self, pe, d=32, h=2):
        super().__init__()
        self.pe, self.h = pe, h
        self.emb = nn.Embedding(V, d)
        self.register_buffer("sin", sinus_pe(MAKS, d))
        self.qkv, self.o = nn.Linear(d, 3 * d), nn.Linear(d, d)
        self.ln1, self.ln2, self.ln3 = nn.LayerNorm(d), nn.LayerNorm(d), nn.LayerNorm(d)
        self.ffn = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
        self.chiq = nn.Linear(d, V)

    def forward(self, X):
        B, L = X.shape
        x = self.emb(X)
        if self.pe == "sinus":
            x = x + self.sin[:L]
        q, k, v = (self.qkv(self.ln1(x)).reshape(B, L, 3, self.h, -1)
                   .permute(2, 0, 3, 1, 4))
        if self.pe == "rope":
            p = torch.arange(L)
            q, k = rope(q, p), rope(k, p)
        a = F.scaled_dot_product_attention(q, k, v)
        x = x + self.o(a.transpose(1, 2).reshape(B, L, -1))
        x = x + self.ffn(self.ln2(x))
        return self.chiq(self.ln3(x))


def batch(n, L, g):
    X = torch.randint(0, V, (n, L), generator=g)
    Y = torch.full((n, L), -100)
    Y[:, SILJISH:] = X[:, :-SILJISH]
    return X, Y


def orgat(pe, seed, qadam=400):
    torch.manual_seed(seed)
    m = Model(pe)
    opt = torch.optim.Adam(m.parameters(), lr=0.003)
    g = torch.Generator().manual_seed(seed)
    for _ in range(qadam):
        X, Y = batch(64, L_OQUV, g)
        loss = F.cross_entropy(m(X).reshape(-1, V), Y.reshape(-1))
        opt.zero_grad()
        loss.backward()
        opt.step()
    m.eval()
    return m


def olcha(m, L):
    X, Y = batch(500, L, torch.Generator().manual_seed(100 + L))
    with torch.no_grad():
        togri = m(X).argmax(-1) == Y
    tashqi = togri[:, L_OQUV:].float().mean().item() if L > L_OQUV else float("nan")
    return togri[:, SILJISH:].float().mean().item(), tashqi


def nishon_ogirligi(m, L):
    """Har pozitsiyada (eng yaxshi bosh) p-3 kalitiga berilgan attention og'irligi."""
    X, _ = batch(200, L, torch.Generator().manual_seed(7))
    with torch.no_grad():
        x = m.emb(X)
        q, k, _ = (m.qkv(m.ln1(x)).reshape(len(X), L, 3, m.h, -1)
                   .permute(2, 0, 3, 1, 4))
        p = torch.arange(L)
        q, k = rope(q, p), rope(k, p)
        w = torch.softmax(q @ k.transpose(-1, -2) / math.sqrt(q.shape[-1]), -1)
    idx = torch.arange(SILJISH, L)
    ogirlik = w[:, :, idx, idx - SILJISH].amax(1)          # (B, pozitsiya)
    ichki = ogirlik[:, :L_OQUV - SILJISH].mean().item()
    tashqi = ogirlik[:, L_OQUV - SILJISH:].mean().item()
    return ichki, tashqi


def main() -> None:
    torch.set_num_threads(1)
    torch.manual_seed(0)
    d = 8
    q, k = torch.randn(d), torch.randn(d)

    print("=== 1. RoPE - vektorni burish ===")
    for p in (0, 1, 10, 100):
        qp = rope(q[None], torch.tensor([p]))[0]
        print(f"  p = {p:>3}: norma {qp.norm():.4f}   birinchi juftlik "
              f"({qp[0]:+.3f}, {qp[1]:+.3f})")
    print(f"  asl norma {q.norm():.4f} - burish uzunlikni o'zgartirmaydi")

    print("\n=== 2. Skor faqat m - n ga bog'liq ===")
    print(f"  {'m':>4} {'n':>4} {'m-n':>4} {'RoPE skor':>10} {'sinus skor':>11}")
    pe = sinus_pe(200, d)
    rope_skor, sinus_skor = [], []
    for m_, n_ in [(5, 2), (25, 22), (105, 102), (7, 2), (107, 102)]:
        qm = rope(q[None], torch.tensor([m_]))[0]
        kn = rope(k[None], torch.tensor([n_]))[0]
        rs = (qm @ kn).item()
        ss = ((q + pe[m_]) @ (k + pe[n_])).item()
        rope_skor.append(rs)
        sinus_skor.append(ss)
        print(f"  {m_:>4} {n_:>4} {m_ - n_:>4} {rs:>10.4f} {ss:>11.4f}")
    oraliq_r = max(rope_skor[:3]) - min(rope_skor[:3])
    oraliq_s = max(sinus_skor[:3]) - min(sinus_skor[:3])
    print(f"  m-n = 3 bo'lgan uch juftlikda skor oralig'i: RoPE {oraliq_r:.1e}, "
          f"qo'shiladigan sinus {oraliq_s:.4f}")
    if oraliq_r < 1e-4 < oraliq_s:
        print("  RoPE: skor AYNAN nisbiy masofaning funksiyasi; sinus: mutlaq p ham kiradi")

    print("\n=== 3. 3-misoldagi vazifa: sinus va RoPE (400 qadam, 3 seed) ===")
    natija = {"sinus": [], "rope": []}
    modellar = {}
    for s in range(3):
        for pe_turi in natija:
            m = orgat(pe_turi, s)
            modellar[(pe_turi, s)] = m
            natija[pe_turi].append([olcha(m, L) for L in (16, 32, 64)])
    r = {k_: np.array(v) for k_, v in natija.items()}
    print(f"  {'pe':<6} {'L=16':>7} {'L=32':>7} {'L=64':>7} {'p>=16 (L=64)':>13}")
    for nom in r:
        print(f"  {nom:<6}" + "".join(f"{r[nom][:, j, 0].mean():>8.3f}" for j in range(3))
              + f"{r[nom][:, 2, 1].mean():>14.3f}")

    print("\n=== 4. Juftlashgan farq, p >= 16 (L=64): RoPE - sinus ===")
    f = r["rope"][:, 2, 1] - r["sinus"][:, 2, 1]
    se = f.std(ddof=1) / np.sqrt(len(f))
    print(f"  seedlar bo'yicha RoPE: {np.round(r['rope'][:, 2, 1], 3).tolist()}")
    print(f"  farq {f.mean():+.4f}, SE {se:.4f}")
    if f.mean() > 2 * se:
        print("  RoPE sezilarli yaxshi ekstrapolyatsiya qiladi")
    elif abs(f.mean()) <= 2 * se:
        print("  farq 2*SE ichida - bu tajribada RoPE ustunligi isbotlanmadi")
    m = modellar[("rope", 0)]
    ich16, _ = nishon_ogirligi(m, 16)
    ich64, tash64 = nishon_ogirligi(m, 64)
    print(f"  RoPE (seed 0): p-3 kalitiga og'irlik  L=16: {ich16:.3f}   "
          f"L=64, p<16: {ich64:.3f}   L=64, p>=16: {tash64:.3f}")
    if tash64 < ich16 - 0.05:
        print("  uzun ketma-ketlikda og'irlik ko'p kalitga yoyiladi - "
              "RoPE ham L=16 dagidek ishlamaydi")
    print("  ⭐ RoPE nisbiy pozitsiyani skorning o'ziga kiritadi - ekstrapolyatsiya "
          "yaxshilanadi, kafolatlanmaydi")


if __name__ == "__main__":
    main()

=== 1. RoPE - vektorni burish ===
  p =   0: norma 3.3953   birinchi juftlik (+1.541, -0.293)
  p =   1: norma 3.3953   birinchi juftlik (+1.080, +1.138)
  p =  10: norma 3.3953   birinchi juftlik (-1.453, -0.592)
  p = 100: norma 3.3953   birinchi juftlik (+1.180, -1.033)
  asl norma 3.3953 - burish uzunlikni o'zgartirmaydi

=== 2. Skor faqat m - n ga bog'liq ===
     m    n  m-n  RoPE skor  sinus skor
     5    2    3     1.1783      4.1280
    25   22    3     1.1783     -0.9895
   105  102    3     1.1783      4.3274
     7    2    5     0.6681      3.8318
   107  102    5     0.6681      4.2476
  m-n = 3 bo'lgan uch juftlikda skor oralig'i: RoPE 7.2e-07, qo'shiladigan sinus 5.3169
  RoPE: skor AYNAN nisbiy masofaning funksiyasi; sinus: mutlaq p ham kiradi

=== 3. 3-misoldagi vazifa: sinus va RoPE (400 qadam, 3 seed) ===
  pe        L=16    L=32    L=64  p>=16 (L=64)
  sinus    1.000   0.561   0.391         0.235
  rope     1.000   0.740   0.531         0.510

=== 4. Juftlashgan farq, p >= 16 (L=64): RoPE - sinus ===
  seedlar bo'yicha RoPE: [0.637, 0.373, 0.519]
  farq +0.2747, SE 0.0629
  RoPE sezilarli yaxshi ekstrapolyatsiya qiladi
  RoPE (seed 0): p-3 kalitiga og'irlik  L=16: 0.852   L=64, p<16: 0.528   L=64, p>=16: 0.444
  uzun ketma-ketlikda og'irlik ko'p kalitga yoyiladi - RoPE ham L=16 dagidek ishlamaydi
  ⭐ RoPE nisbiy pozitsiyani skorning o'ziga kiritadi - ekstrapolyatsiya yaxshilanadi, kafolatlanmaydi

x = self.emb(X)                                                # ⚠️
x = self.emb(X) + self.pe[:X.shape[1]]                         # ✅

x = self.emb(X) + self.pe[:X.shape[0]]      # batch o'lchami!  # ⚠️
x = self.emb(X) + self.pe[:X.shape[1]]      # ketma-ketlik uzunligi # ✅

self.pe = sinus_pe(512, d)                  # .to("cuda") da qoladi # ⚠️
self.register_buffer("pe", sinus_pe(512, d))                   # ✅

self.pe = nn.Parameter(sinus_pe(512, d))    # endi bu sinus emas   # ⚠️
self.register_buffer("pe", sinus_pe(512, d))  # yoki ongli nn.Embedding # ✅

self.pos(torch.arange(L))                   # L > maks: IndexError  # ⚠️
assert L <= self.pos.num_embeddings, "o'quvdan uzun kirish"       # ✅

q, k, v = rope(q, p), rope(k, p), rope(v, p)                   # ⚠️
q, k = rope(q, p), rope(k, p)               # v o'zgarmaydi        # ✅

aniq = baholash(model, test_L16)            # faqat o'quv uzunligi  # ⚠️
for L in (16, 32, 64): print(L, baholash(model, test(L)))         # ✅

1.  # self-attention da qatorlar almashtirilsa chiqish nima bo'ladi?
2.  # o'rtacha pooling dan keyin-chi?
3.  # PE[p, 2i] va PE[p, 2i+1] formulalari?
4.  # d = 16 da PE[p] normasi?
5.  # eng qisqa to'lqin uzunligi?
6.  # PE[p + k] va PE[p] qanday bog'langan?
7.  # PE[p] . PE[p + k] nimaga bog'liq?
8.  # o'rganiladigan jadvalda maks_uzunlik = 512, d = 768 - parametrlar?
9.  # jadvaldan uzun ketma-ketlik berilsa?
10. # RoPE q va k ga nima qiladi?
11. # RoPE da q_m' . k_n' nimaga bog'liq?
12. # RoPE v ga qo'llanadimi?

1.  x = self.emb(X) + self.pe[:X.shape[0]]

2.  class Model(nn.Module):
        def __init__(self):
            super().__init__()
            self.pe = sinus_pe(512, 64)

3.  self.pos = nn.Embedding(128, d)
    x = self.emb(X) + self.pos(torch.arange(X.shape[1]))   # X uzunligi 300

4.  q, k, v = rope(q, p), rope(k, p), rope(v, p)

5.  print("uzun matnlarga tayyor:", baholash(model, test_L16))