4.8-dars: Unicode nima va nega muhim

print(len("salom"))                 # 5
print(len("салом"))                 # 5
print(len("O'zbekiston"))           # 11

print("Ali" == "Аli")               # False — nega?!
print("é" == "é")             # False — nega?!
print("ı".upper())                  # 'I' — turkcha noto'g'ri
print("SALOM".lower() == "salom")   # True
print("İSTANBUL".lower())           # 'i̇stanbul' — qo'shimcha nuqta!

UnicodeEncodeError: 'charmap' codec can't encode character 'ʻ'
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd0 in position 0

0-31     boshqaruv belgilari (\n, \t, ...)
32-47    bo'sh joy va tinish
48-57    raqamlar 0-9
65-90    A-Z
97-122   a-z

Bayt 0xC0:
  CP1251 (kirillitsa) → 'А'
  CP1252 (g'arbiy)    → 'À'
  CP1254 (turk)       → 'À'

Привет  →  noto'g'ri sahifada  →  Ïðèâåò

U+0041  →  A       LATIN CAPITAL LETTER A
U+0430  →  а       CYRILLIC SMALL LETTER A
U+4E2D  →  中       CJK UNIFIED IDEOGRAPH-4E2D
U+1F60A →  😊      SMILING FACE WITH SMILING EYES

Oraliq:       U+0000 – U+10FFFF
Jami:         1 114 112 pozitsiya
Belgilangan:  ~150 000 belgi (Unicode 16.0)
Bo'sh:        kelajak uchun

Unicode:  'А' = U+0410  ← belgi ↔ raqam
UTF-8:    U+0410 = 0xD0 0x90  ← raqam ↔ baytlar

matn = "Аа中😊"

for belgi in matn:
    print(f"  {belgi}  U+{ord(belgi):04X}  {ord(belgi):>7}")

  А  U+0410     1040
  а  U+0430     1072
  中  U+4E2D    20013
  😊  U+1F60A   128522

U+0000–007F   ASCII
U+0080–00FF   Latin-1 qo'shimcha (é, ñ, ü)
U+0100–017F   Latin kengaytirilgan-A (ā, ć, ę)
U+0370–03FF   Yunon
U+0400–04FF   Kirillitsa       ← o'zbek kirillchasi
U+0530–058F   Arman
U+0590–05FF   Ibroniy
U+0600–06FF   Arab
U+4E00–9FFF   CJK (xitoy, yapon, koreys)

import sys

print(sys.getsizeof("abc"))         # 52  — 1 bayt/belgi
print(sys.getsizeof("абв"))         # 78  — 2 bayt/belgi (BMP)
print(sys.getsizeof("ab😊"))        # 92  — 4 bayt/belgi (BMP dan tashqarida)

namunalar = [
    ("a",           "oddiy harf"),
    ("é",           "tayyor belgi (NFC)"),
    ("é",     "e + urg'u (NFD)"),
    ("👨‍👩‍👧‍👦",     "oila emojisi"),
    ("🇺🇿",         "bayroq"),
    ("é̂",     "ikki qo'shimcha belgi"),
    ("ᄀᄀᄀ각",     "koreys hangul"),
]

for matn, izoh in namunalar:
    print(f"  {matn:<8} len={len(matn)}  {izoh}")

  a        len=1  oddiy harf
  é        len=1  tayyor belgi (NFC)
  é        len=2  e + urg'u (NFD)
  👨‍👩‍👧‍👦        len=7  oila emojisi
  🇺🇿       len=2  bayroq
  é̂        len=3  ikki qo'shimcha belgi
  ᄀᄀᄀ각      len=4  koreys hangul

bayroq = "🇺🇿"
for c in bayroq:
    import unicodedata
    print(f"  U+{ord(c):05X}  {unicodedata.name(c)}")

  U+1F1FA  REGIONAL INDICATOR SYMBOL LETTER U
  U+1F1FF  REGIONAL INDICATOR SYMBOL LETTER Z

matn = "Salom 🇺🇿"
print(len(matn))                # 8
print(matn[:7])                 # 'Salom 🇺' — bayroq buzildi

pip install grapheme

import grapheme
print(grapheme.length("👨‍👩‍👧‍👦"))       # 1
print(grapheme.slice("Salom 🇺🇿", 0, 7))  # 'Salom 🇺🇿'

a = "é"                 # U+00E9 — tayyor belgi
b = "é"           # U+0065 U+0301 — e + urg'u

print(a, b)             # é é — bir xil ko'rinadi
print(a == b)           # False!
print(len(a), len(b))   # 1 2

import unicodedata

print(unicodedata.normalize("NFC", b) == a)     # True
print(unicodedata.normalize("NFD", a) == b)     # True

matn = "café"
nfc = unicodedata.normalize("NFC", matn)
nfd = unicodedata.normalize("NFD", matn)

print(f"  Asl:  {len(matn)} belgi")
print(f"  NFC:  {len(nfc)} belgi")
print(f"  NFD:  {len(nfd)} belgi")
print(f"  NFC == NFD: {nfc == nfd}")

  Asl:  4 belgi
  NFC:  4 belgi
  NFD:  5 belgi
  NFC == NFD: False

namunalar = ["①", "fi", "㎏", "½", "Ⅷ", "２", "ff"]

print(f"  {'Asl':<6} {'NFC':<8} {'NFKC':<10} Izoh")
print("  " + "─" * 44)
for m in namunalar:
    nfc = unicodedata.normalize("NFC", m)
    nfkc = unicodedata.normalize("NFKC", m)
    print(f"  {m:<6} {nfc:<8} {nfkc:<10} {unicodedata.name(m)[:30]}")

  Asl    NFC      NFKC       Izoh
  ────────────────────────────────────────────
  ①      ①        1          CIRCLED DIGIT ONE
  fi      fi        fi         LATIN SMALL LIGATURE FI
  ㎏     ㎏       kg         SQUARE KG
  ½      ½        1⁄2        VULGAR FRACTION ONE HALF
  Ⅷ      Ⅷ        VIII       ROMAN NUMERAL EIGHT
  ２     ２       2          FULLWIDTH DIGIT TWO
  ff      ff        ff         LATIN SMALL LIGATURE FF

# Foydalanuvchi "２０２６" (to'liq kenglikda) yozdi
kirish = "２０２６"
print(kirish.isdecimal())                       # True
print(int(kirish))                              # 2026 — Python o'zi tushundi

# Lekin qidiruvda muammo:
print("2026" in "２０２６ yil")                   # False
normal = unicodedata.normalize("NFKC", "２０２６ yil")
print("2026" in normal)                         # True ✅

# macOS da yaratilgan fayl: "café.txt"
# Aslida: "cafe" + U+0301 + ".txt"

nom = "café.txt"                                # NFC
# open(nom)                                     # ❌ FileNotFoundError

nom_nfd = unicodedata.normalize("NFD", nom)
# open(nom_nfd)                                 # ✅

import unicodedata

for c in "Aa1 !á中😊\n":
    print(f"  {c!r:<8} {unicodedata.category(c):<4} "
          f"{unicodedata.name(c, '(nomsiz)')}")

  'A'      Lu   LATIN CAPITAL LETTER A
  'a'      Ll   LATIN SMALL LETTER A
  '1'      Nd   DIGIT ONE
  ' '      Zs   SPACE
  '!'      Po   EXCLAMATION MARK
  'á'      Ll   LATIN SMALL LETTER A WITH ACUTE
  '中'      Lo   CJK UNIFIED IDEOGRAPH-4E2D
  '😊'     So   SMILING FACE WITH SMILING EYES
  '\n'     Cc   (nomsiz)

L — harf      N — son       P — tinish
M — belgi     S — simvol    Z — ajratgich    C — boshqaruv

def faqat_harf(matn: str) -> str:
    """Faqat harflarni qoldiradi."""
    return "".join(c for c in matn if unicodedata.category(c).startswith("L"))


def urgusiz(matn: str) -> str:
    """Urg'u belgilarini olib tashlaydi."""
    nfd = unicodedata.normalize("NFD", matn)
    return "".join(c for c in nfd if unicodedata.category(c) != "Mn")


print(faqat_harf("Salom, dunyo! 123"))          # 'Salomdunyo'
print(urgusiz("café résumé naïve"))             # 'cafe resume naive'
print(urgusiz("Ўзбекистон"))                    # 'Узбекистон' — ў → у!

apostroflar = [
    ("'",  "U+0027", "APOSTROPHE — klaviaturadagi oddiy"),
    ("ʻ",  "U+02BB", "MODIFIER LETTER TURNED COMMA — RASMIY"),
    ("ʼ",  "U+02BC", "MODIFIER LETTER APOSTROPHE"),
    ("‘",  "U+2018", "LEFT SINGLE QUOTATION MARK"),
    ("’",  "U+2019", "RIGHT SINGLE QUOTATION MARK — Word avtomatik"),
    ("`",  "U+0060", "GRAVE ACCENT"),
]

for belgi, kod, izoh in apostroflar:
    print(f"  {belgi}  {kod}  {izoh}")

variantlar = ["O'zbekiston", "Oʻzbekiston", "O’zbekiston", "Ozbekiston"]

print("  Barcha juftlikni solishtirish:")
for i, a in enumerate(variantlar):
    for b in variantlar[i+1:]:
        print(f"    {a!r} == {b!r}  →  {a == b}")

APOSTROFLAR = "'ʻʼ‘’`´"

def uz_normalize(matn: str) -> str:
    """O'zbekcha matnni solishtirish uchun normallashtiradi."""
    matn = unicodedata.normalize("NFC", matn)
    # Barcha apostrof turini bittasiga keltirish
    jadval = str.maketrans({a: "'" for a in APOSTROFLAR})
    return matn.translate(jadval).casefold()


for v in variantlar:
    print(f"  {v!r:<18} → {uz_normalize(v)!r}")

  "O'zbekiston"      → "o'zbekiston"
  'Oʻzbekiston'      → "o'zbekiston"
  'O’zbekiston'      → "o'zbekiston"
  'Ozbekiston'       → 'ozbekiston'

def uz_qidiruv_kaliti(matn: str) -> str:
    """Qidiruv uchun — apostroflarni butunlay olib tashlaydi."""
    normal = uz_normalize(matn)
    return normal.replace("'", "")


for v in variantlar:
    print(f"  {v!r:<18} → {uz_qidiruv_kaliti(v)!r}")

KIRIL_LOTIN = {
    "а":"a","б":"b","в":"v","г":"g","д":"d","е":"e","ё":"yo","ж":"j",
    "з":"z","и":"i","й":"y","к":"k","л":"l","м":"m","н":"n","о":"o",
    "п":"p","р":"r","с":"s","т":"t","у":"u","ф":"f","х":"x","ц":"ts",
    "ч":"ch","ш":"sh","ъ":"'","ь":"","э":"e","ю":"yu","я":"ya",
    "ў":"o'","қ":"q","ғ":"g'","ҳ":"h",
}


def kiril_lotin(matn: str) -> str:
    natija = []
    for c in matn:
        kichik = c.lower()
        if kichik in KIRIL_LOTIN:
            almashtirilgan = KIRIL_LOTIN[kichik]
            natija.append(almashtirilgan.capitalize() if c.isupper()
                          else almashtirilgan)
        else:
            natija.append(c)
    return "".join(natija)


print(kiril_lotin("Ўзбекистон Республикаси"))
print(kiril_lotin("Тошкент шаҳри"))

O'zbekiston Respublikasi
Toshkent shahri

Ingliz:  i → I,  I → i
Turk:    i → İ,  I → ı     (nuqtali va nuqtasiz)

print("i".upper())          # 'I'   — turkchada 'İ' bo'lishi kerak
print("I".lower())          # 'i'   — turkchada 'ı' bo'lishi kerak
print("İ".lower())          # 'i̇'   — ikki kod nuqtasi!
print(len("İ".lower()))     # 2

pip install PyICU

print("ß".upper())          # 'SS'  — bitta belgi ikkitaga
print("ß".lower())          # 'ß'
print("ß".casefold())       # 'ss'
print("STRASSE".casefold() == "straße".casefold())      # True ✅
print("STRASSE".lower() == "straße".lower())            # False

print("ΣΊΣΥΦΟΣ".lower())        # 'σίσυφος' — oxirgi Σ → ς
print("ΣΊΣΥΦΟΣ".casefold())     # 'σίσυφοσ' — barchasi σ

namunalar = ["ß", "fi", "ǰ", "ffl"]
for m in namunalar:
    print(f"  {m!r} ({len(m)}) → upper: {m.upper()!r} ({len(m.upper())})")

  'ß' (1) → upper: 'SS' (2)
  'fi' (1) → upper: 'FI' (2)
  'ǰ' (1) → upper: 'J̌' (2)
  'ffl' (1) → upper: 'FFL' (3)

XAVFLI = {
    "​": "ZERO WIDTH SPACE",
    "‌": "ZERO WIDTH NON-JOINER",
    "‍": "ZERO WIDTH JOINER",
    "": "ZERO WIDTH NO-BREAK SPACE (BOM)",
    " ": "NO-BREAK SPACE",
    "⁠": "WORD JOINER",
    "᠎": "MONGOLIAN VOWEL SEPARATOR",
}

matn = "sa​lom"
print(f"  {matn!r}")
print(f"  Ko'rinishi: {matn}")
print(f"  len: {len(matn)}")
print(f"  == 'salom': {matn == 'salom'}")

YONALISH = {
    "‪": "LEFT-TO-RIGHT EMBEDDING",
    "‫": "RIGHT-TO-LEFT EMBEDDING",
    "‭": "LEFT-TO-RIGHT OVERRIDE",
    "‮": "RIGHT-TO-LEFT OVERRIDE",   # ← eng xavflisi
    "⁦": "LEFT-TO-RIGHT ISOLATE",
    "⁩": "POP DIRECTIONAL ISOLATE",
}

# Fayl nomini yashirish hujumi
fayl = "hisobot‮gnp.exe"
print(f"  Ko'rinishi: {fayl}")
print(f"  Aslida:     {fayl!r}")

# Ko'rinadigan kod va bajariladigan kod har xil bo'lishi mumkin

def xavfli_belgilarni_tekshir(matn: str) -> list[str]:
    """Xavfli Unicode belgilarini topadi."""
    topilgan = []
    for i, c in enumerate(matn):
        kat = unicodedata.category(c)
        if kat == "Cf":         # Format belgilari
            nom = unicodedata.name(c, f"U+{ord(c):04X}")
            topilgan.append(f"{i}-pozitsiya: {nom}")
    return topilgan


print(xavfli_belgilarni_tekshir("hisobot‮gnp.exe"))
print(xavfli_belgilarni_tekshir("sa​lom"))

juftliklar = [
    ("a", "а", "lotin a va kirillcha а"),
    ("o", "о", "lotin o va kirillcha о"),
    ("p", "р", "lotin p va kirillcha р"),
    ("e", "е", "lotin e va kirillcha е"),
    ("c", "с", "lotin c va kirillcha с"),
]

for lot, kir, izoh in juftliklar:
    print(f"  {lot} vs {kir}: teng={lot == kir}  "
          f"U+{ord(lot):04X} vs U+{ord(kir):04X}  {izoh}")

Unicode        — belgi ↔ raqam jadvali (U+0000 – U+10FFFF)
Kod nuqtasi    — bitta raqam
Grafema        — foydalanuvchi ko'radigan bitta belgi
Kodlash        — kod nuqtasi ↔ baytlar (UTF-8 — keyingi dars)

import unicodedata

unicodedata.normalize("NFC", matn)      birlashtiradi  ← SAQLASH uchun
unicodedata.normalize("NFD", matn)      ajratadi
unicodedata.normalize("NFKC", matn)     + moslik       ← QIDIRUV uchun
unicodedata.normalize("NFKD", matn)

unicodedata.name(c)             belgi nomi
unicodedata.category(c)         kategoriya (Lu, Ll, Nd, Mn, ...)
unicodedata.numeric(c)          sonli qiymat (½ → 0.5)
ord(c)  chr(n)                  kod nuqtasi

# Urg'usiz
"".join(c for c in unicodedata.normalize("NFD", m)
        if unicodedata.category(c) != "Mn")

# Faqat harflar
"".join(c for c in m if unicodedata.category(c).startswith("L"))

# Qidiruv kaliti
unicodedata.normalize("NFKC", m).casefold()

Apostrof turlari: ' ʻ ʼ ‘ ’ ` — barchasi TURLI belgi
Rasmiy: ʻ (U+02BB)
Solishtirishdan oldin bittasiga keltiring

Cf kategoriyasi: ​ ‌ ‍  ⁠   — ko'rinmas
U+202E (RLO)                — matn yo'nalishini teskarilaydi
Homoglifiya: a/а, o/о, p/р  — turli belgi, bir xil ko'rinish

"""Har qanday matnni Unicode nuqtai nazaridan tahlil qilish."""

import unicodedata
import sys


def tahlil(matn: str) -> None:
    print(f"\n┌─ {matn!r}")
    print(f"│  Ko'rinishi:  {matn}")
    print(f"│  len():       {len(matn)}")
    print(f"│  Xotira:      {sys.getsizeof(matn)} bayt")
    print(f"│  UTF-8:       {len(matn.encode('utf-8'))} bayt")
    print("│")
    print(f"│  {'#':>3} {'Belgi':<6} {'Kod':<10} {'Kat':<4} {'Nomi'}")
    print(f"│  {'─' * 62}")

    for i, c in enumerate(matn):
        nom = unicodedata.name(c, "(nomsiz)")
        kat = unicodedata.category(c)
        korinish = c if kat[0] not in "CZ" else "·"
        print(f"│  {i:>3} {korinish:<6} U+{ord(c):05X}   {kat:<4} {nom[:38]}")

    # Normalizatsiya
    shakllar = {s: unicodedata.normalize(s, matn) for s in ["NFC", "NFD", "NFKC", "NFKD"]}
    print("│")
    print("│  Normalizatsiya:")
    for shakl, natija in shakllar.items():
        farq = "" if natija == matn else "  ← o'zgardi"
        print(f"│    {shakl}: {natija!r} ({len(natija)}){farq}")
    print("└─")


for matn in ["salom", "café", "café", "O'zbekiston", "Oʻzbekiston",
             "😊", "🇺🇿", "½ ① ㎏"]:
    tahlil(matn)

┌─ 'café'
│  Ko'rinishi:  café
│  len():       4
│  Xotira:      61 bayt
│  UTF-8:       5 bayt
│
│    # Belgi  Kod        Kat  Nomi
│  ──────────────────────────────────────────────────────────────
│    0 c      U+00063   Ll   LATIN SMALL LETTER C
│    1 a      U+00061   Ll   LATIN SMALL LETTER A
│    2 f      U+00066   Ll   LATIN SMALL LETTER F
│    3 é      U+000E9   Ll   LATIN SMALL LETTER E WITH ACUTE
│
│  Normalizatsiya:
│    NFC: 'café' (4)
│    NFD: 'café' (5)  ← o'zgardi
│    NFKC: 'café' (4)
│    NFKD: 'café' (5)  ← o'zgardi
└─

┌─ '½ ① ㎏'
│  ...
│  Normalizatsiya:
│    NFC: '½ ① ㎏' (5)
│    NFD: '½ ① ㎏' (5)
│    NFKC: '1⁄2 1 kg' (8)  ← o'zgardi
│    NFKD: '1⁄2 1 kg' (8)  ← o'zgardi
└─

"""Nega normalizatsiyasiz qidiruv ishlamaydi."""

import unicodedata

BAZA = [
    "José García",
    "José García",       # NFD
    "JOSÉ GARCÍA",
    "jose garcia",
    "Café Central",
    "Café Central",       # NFD
    "CAFÉ CENTRAL",
]


def oddiy_qidiruv(sorov: str, baza: list[str]) -> list[str]:
    return [x for x in baza if sorov in x]


def yaxshi_qidiruv(sorov: str, baza: list[str]) -> list[str]:
    """NFKC + casefold."""
    def kalit(s: str) -> str:
        return unicodedata.normalize("NFKC", s).casefold()

    k = kalit(sorov)
    return [x for x in baza if k in kalit(x)]


def eng_yaxshi_qidiruv(sorov: str, baza: list[str]) -> list[str]:
    """NFKC + casefold + urg'usiz."""
    def kalit(s: str) -> str:
        nfd = unicodedata.normalize("NFD", s)
        urgusiz = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
        return unicodedata.normalize("NFKC", urgusiz).casefold()

    k = kalit(sorov)
    return [x for x in baza if k in kalit(x)]


sorovlar = ["José", "jose", "JOSÉ", "café", "cafe", "CAFE"]

for sorov in sorovlar:
    oddiy = oddiy_qidiruv(sorov, BAZA)
    yaxshi = yaxshi_qidiruv(sorov, BAZA)
    eng = eng_yaxshi_qidiruv(sorov, BAZA)

    print(f"\n  Qidiruv: {sorov!r}")
    print(f"    Oddiy:      {len(oddiy)} ta topildi")
    print(f"    NFKC+fold:  {len(yaxshi)} ta topildi")
    print(f"    + urg'usiz: {len(eng)} ta topildi")


print("\n\n=== Urg'uni olib tashlash ===")
namunalar = ["café", "résumé", "naïve", "Ўзбекистон", "Müller", "São Paulo"]

def urgusiz(matn: str) -> str:
    nfd = unicodedata.normalize("NFD", matn)
    return "".join(c for c in nfd if unicodedata.category(c) != "Mn")

for m in namunalar:
    print(f"  {m:<16} → {urgusiz(m)}")

print("\n  ⚠️ Diqqat: kirillcha 'ў' ham 'у' ga aylandi.")
print("  O'zbek kirillchasi uchun bu noto'g'ri bo'lishi mumkin.")


print("\n\n=== NFKC — moslik almashtirish ===")
kirishlar = [
    ("２０２６", "to'liq kenglikdagi raqamlar"),
    ("①②③", "doiradagi raqamlar"),
    ("file", "ligatura"),
    ("１０ ㎏", "to'liq kenglik + kvadrat kg"),
    ("Ⅷ", "rim raqami"),
]

for kirish, izoh in kirishlar:
    nfkc = unicodedata.normalize("NFKC", kirish)
    print(f"  {kirish:<10} → {nfkc:<12} {izoh}")

  Qidiruv: 'José'
    Oddiy:      2 ta topildi
    NFKC+fold:  3 ta topildi
    + urg'usiz: 4 ta topildi

  Qidiruv: 'cafe'
    Oddiy:      0 ta topildi
    NFKC+fold:  0 ta topildi
    + urg'usiz: 3 ta topildi

=== Urg'uni olib tashlash ===
  café             → cafe
  résumé           → resume
  naïve            → naive
  Ўзбекистон       → Узбекистон
  Müller           → Muller
  São Paulo        → Sao Paulo

=== NFKC — moslik almashtirish ===
  ２０２６       → 2026         to'liq kenglikdagi raqamlar
  ①②③        → 123          doiradagi raqamlar
  file        → file         ligatura
  １０ ㎏      → 10 kg        to'liq kenglik + kvadrat kg
  Ⅷ          → VIII         rim raqami

"""O'zbekcha matnni to'g'ri qayta ishlash."""

import unicodedata

APOSTROFLAR = "'ʻʼ‘’`´ʹ׳"
RASMIY_APOSTROF = "ʻ"       # U+02BB


def uz_saqlash(matn: str) -> str:
    """Ma'lumotlar bazasiga saqlash uchun — rasmiy apostrof."""
    matn = unicodedata.normalize("NFC", matn)
    jadval = str.maketrans({a: RASMIY_APOSTROF for a in APOSTROFLAR})
    return " ".join(matn.translate(jadval).split())


def uz_solishtirish(matn: str) -> str:
    """Solishtirish uchun — bitta apostrof + kichik harf."""
    return uz_saqlash(matn).replace(RASMIY_APOSTROF, "'").casefold()


def uz_qidiruv(matn: str) -> str:
    """Qidiruv uchun — apostrofsiz."""
    return uz_solishtirish(matn).replace("'", "")


VARIANTLAR = [
    "O'zbekiston",          # oddiy apostrof
    "Oʻzbekiston",          # rasmiy (U+02BB)
    "O’zbekiston",          # Word avtomatik (U+2019)
    "O‘zbekiston",          # chap tirnoq (U+2018)
    "OʼZBEKISTON",          # U+02BC, katta harf
    "Ozbekiston",           # apostrofsiz
    "  o'zbekiston  ",      # bo'shliq bilan
]

print("┌─ APOSTROF VARIANTLARI")
print("│")
print(f"│  {'Asl':<20} {'U+':<8} {'Saqlash':<16} {'Solishtirish':<16} {'Qidiruv'}")
print(f"│  {'─' * 78}")

for v in VARIANTLAR:
    # Apostrof kodini topish
    apostrof_kodi = ""
    for c in v:
        if c in APOSTROFLAR:
            apostrof_kodi = f"U+{ord(c):04X}"
            break

    print(f"│  {v!r:<20} {apostrof_kodi:<8} "
          f"{uz_saqlash(v)!r:<16} "
          f"{uz_solishtirish(v)!r:<16} "
          f"{uz_qidiruv(v)!r}")

print("└─")

print("\n=== Guruhlash ===")
from collections import defaultdict

guruhlar = defaultdict(list)
for v in VARIANTLAR:
    guruhlar[uz_qidiruv(v)].append(v)

for kalit, a_zolar in guruhlar.items():
    print(f"  {kalit!r}: {len(a_zolar)} ta variant")
    for a in a_zolar:
        print(f"    - {a!r}")


print("\n\n=== Kirillcha ↔ lotincha ===")

KIRIL_LOTIN = {
    "а":"a","б":"b","в":"v","г":"g","д":"d","е":"e","ё":"yo","ж":"j",
    "з":"z","и":"i","й":"y","к":"k","л":"l","м":"m","н":"n","о":"o",
    "п":"p","р":"r","с":"s","т":"t","у":"u","ф":"f","х":"x","ц":"ts",
    "ч":"ch","ш":"sh","щ":"sh","ъ":"ʼ","ь":"","э":"e","ю":"yu","я":"ya",
    "ў":"oʻ","қ":"q","ғ":"gʻ","ҳ":"h",
}


def kiril_lotin(matn: str) -> str:
    natija = []
    for c in matn:
        kichik = c.lower()
        if kichik in KIRIL_LOTIN:
            almash = KIRIL_LOTIN[kichik]
            natija.append(almash.capitalize() if c.isupper() else almash)
        else:
            natija.append(c)
    return "".join(natija)


matnlar = [
    "Ўзбекистон Республикаси",
    "Тошкент шаҳри",
    "Ғафур Ғулом кўчаси",
    "Қашқадарё вилояти",
]

for m in matnlar:
    print(f"  {m:<28} → {kiril_lotin(m)}")


print("\n\n=== ⚠️ Homoglifiya xavfi ===")
juftliklar = [("a", "а"), ("o", "о"), ("p", "р"), ("e", "е"), ("c", "с"),
              ("y", "у"), ("x", "х")]

print(f"  {'Lotin':<8} {'Kirill':<8} {'Teng?':<8} Kodlar")
print("  " + "─" * 42)
for lot, kir in juftliklar:
    print(f"  {lot:<8} {kir:<8} {str(lot == kir):<8} "
          f"U+{ord(lot):04X} / U+{ord(kir):04X}")

print("\n  Misol: 'ассount' — 'a' va 'c' kirillcha")
soxta = "ассount"
haqiqiy = "account"
print(f"    Soxta:   {soxta!r}")
print(f"    Haqiqiy: {haqiqiy!r}")
print(f"    Ko'rinishi bir xil: {soxta} vs {haqiqiy}")
print(f"    Teng? {soxta == haqiqiy}")

┌─ APOSTROF VARIANTLARI
│
│  Asl                  U+       Saqlash          Solishtirish     Qidiruv
│  ──────────────────────────────────────────────────────────────────────────────
│  "O'zbekiston"        U+0027   'Oʻzbekiston'    "o'zbekiston"    'ozbekiston'
│  'Oʻzbekiston'        U+02BB   'Oʻzbekiston'    "o'zbekiston"    'ozbekiston'
│  'O’zbekiston'        U+2019   'Oʻzbekiston'    "o'zbekiston"    'ozbekiston'
│  'O‘zbekiston'        U+2018   'Oʻzbekiston'    "o'zbekiston"    'ozbekiston'
│  'OʼZBEKISTON'        U+02BC   'OʻZBEKISTON'    "o'zbekiston"    'ozbekiston'
│  'Ozbekiston'                  'Ozbekiston'     'ozbekiston'     'ozbekiston'
│  "  o'zbekiston  "    U+0027   'oʻzbekiston'    "o'zbekiston"    'ozbekiston'
└─

=== Guruhlash ===
  'ozbekiston': 7 ta variant
    ...

=== Kirillcha ↔ lotincha ===
  Ўзбекистон Республикаси      → Oʻzbekiston Respublikasi
  Тошкент шаҳри                → Toshkent shahri
  Ғафур Ғулом кўчаси           → Gʻafur Gʻulom koʻchasi
  Қашқадарё вилояти            → Qashqadaryo viloyati

=== ⚠️ Homoglifiya xavfi ===
  Lotin    Kirill   Teng?    Kodlar
  ──────────────────────────────────────────
  a        а        False    U+0061 / U+0430
  o        о        False    U+006F / U+043E
  ...

"""Unicode xavfsizlik tekshiruvi."""

import unicodedata

# Xavfli kategoriyalar
XAVFLI_KATEGORIYALAR = {
    "Cf": "Format belgisi (ko'rinmas)",
    "Cc": "Boshqaruv belgisi",
    "Co": "Shaxsiy foydalanish",
    "Cs": "Surrogat",
}

# Yo'nalishni o'zgartiruvchi belgilar — eng xavflisi
YONALISH_BELGILARI = {
    0x202A: "LEFT-TO-RIGHT EMBEDDING",
    0x202B: "RIGHT-TO-LEFT EMBEDDING",
    0x202C: "POP DIRECTIONAL FORMATTING",
    0x202D: "LEFT-TO-RIGHT OVERRIDE",
    0x202E: "RIGHT-TO-LEFT OVERRIDE",
    0x2066: "LEFT-TO-RIGHT ISOLATE",
    0x2067: "RIGHT-TO-LEFT ISOLATE",
    0x2068: "FIRST STRONG ISOLATE",
    0x2069: "POP DIRECTIONAL ISOLATE",
}

# Kirillcha ↔ lotincha homoglifiya
HOMOGLIFLAR = {
    "а": "a", "е": "e", "о": "o", "р": "p", "с": "c",
    "у": "y", "х": "x", "А": "A", "В": "B", "Е": "E",
    "К": "K", "М": "M", "Н": "H", "О": "O", "Р": "P",
    "С": "C", "Т": "T", "У": "Y", "Х": "X",
}


def xavfsizlik_tekshiruvi(matn: str) -> dict:
    """Matndagi xavfli Unicode belgilarini topadi."""
    natija = {
        "korinmas": [],
        "yonalish": [],
        "homoglif": [],
        "aralash_yozuv": False,
    }

    yozuvlar = set()

    for i, c in enumerate(matn):
        kod = ord(c)
        kat = unicodedata.category(c)
        nom = unicodedata.name(c, f"U+{kod:04X}")

        # Yo'nalish belgilari
        if kod in YONALISH_BELGILARI:
            natija["yonalish"].append((i, c, YONALISH_BELGILARI[kod]))

        # Ko'rinmas belgilar
        elif kat in ("Cf", "Cc") and c not in "\n\r\t":
            natija["korinmas"].append((i, c, nom))

        # Homogliflar
        if c in HOMOGLIFLAR:
            natija["homoglif"].append((i, c, HOMOGLIFLAR[c]))

        # Yozuv turini aniqlash
        if kat.startswith("L"):
            if "CYRILLIC" in nom:
                yozuvlar.add("kirill")
            elif "LATIN" in nom:
                yozuvlar.add("lotin")
            elif "ARABIC" in nom:
                yozuvlar.add("arab")

    natija["aralash_yozuv"] = len(yozuvlar) > 1
    natija["yozuvlar"] = yozuvlar

    return natija


def hisobot(matn: str, izoh: str = "") -> None:
    n = xavfsizlik_tekshiruvi(matn)
    xavf = bool(n["korinmas"] or n["yonalish"] or n["homoglif"]
                or n["aralash_yozuv"])

    belgi = "🔴 XAVFLI" if xavf else "✅ XAVFSIZ"
    print(f"\n  {belgi}  {matn!r}")
    if izoh:
        print(f"    {izoh}")
    print(f"    Ko'rinishi: {matn}")

    if n["yonalish"]:
        print("    🔴 Yo'nalish belgilari:")
        for i, c, nom in n["yonalish"]:
            print(f"       {i}-pozitsiya: {nom}")

    if n["korinmas"]:
        print("    ⚠️  Ko'rinmas belgilar:")
        for i, c, nom in n["korinmas"]:
            print(f"       {i}-pozitsiya: {nom}")

    if n["homoglif"]:
        print("    ⚠️  Homogliflar (kirillcha, lotincha ko'rinadi):")
        for i, c, lot in n["homoglif"]:
            print(f"       {i}-pozitsiya: {c!r} (U+{ord(c):04X}) ~ {lot!r}")

    if n["aralash_yozuv"]:
        print(f"    ⚠️  Aralash yozuv tizimi: {n['yozuvlar']}")


print("┌─ UNICODE XAVFSIZLIK TEKSHIRUVI")

sinovlar = [
    ("salom", "oddiy matn"),
    ("Salom, dunyo!", "tinish belgilari bilan"),
    ("O'zbekiston", "apostrof bilan"),
    ("sa​lom", "nol kenglikdagi bo'sh joy"),
    ("hisobot‮gnp.exe", "fayl nomini yashirish hujumi"),
    ("аccount", "kirillcha 'а' bilan (homoglif)"),
    ("Тошкент city", "aralash yozuv"),
    ("password", "BOM belgisi"),
]

for matn, izoh in sinovlar:
    hisobot(matn, izoh)

print("\n└─")

print("""

  Amaliy tavsiyalar:

  1. Foydalanuvchi kiritmasini tekshiring — ayniqsa:
     • Fayl nomlari
     • Domen nomlari
     • Foydalanuvchi nomlari
     • Kod (Trojan Source hujumi)

  2. Cf kategoriyasidagi belgilarni olib tashlang yoki rad eting

  3. Aralash yozuv tizimini shubhali deb hisoblang

  4. Ruff RUF001/RUF002/RUF003 qoidalarini yoqing
""")

  🔴 XAVFLI  'hisobot\u202egnp.exe'
    fayl nomini yashirish hujumi
    Ko'rinishi: ...
    🔴 Yo'nalish belgilari:
       7-pozitsiya: RIGHT-TO-LEFT OVERRIDE

  🔴 XAVFLI  'аccount'
    kirillcha 'а' bilan (homoglif)
    Ko'rinishi: аccount
    ⚠️  Homogliflar (kirillcha, lotincha ko'rinadi):
       0-pozitsiya: 'а' (U+0430) ~ 'a'
    ⚠️  Aralash yozuv tizimi: ...

if ism == qidiruv:                                  # ❌
if normalize("NFC", ism) == normalize("NFC", qidiruv):   # ✅

open("café.txt")                                    # ❌ macOS da topilmasligi mumkin

import unicodedata, os
for f in os.listdir("."):                           # ✅
    if unicodedata.normalize("NFC", f) == "café.txt":
        open(f)

matn[:10]                                           # ❌ emoji buzilishi mumkin

import grapheme                                     # ✅
grapheme.slice(matn, 0, 10)

if a.lower() == b.lower():                          # ⚠️ ß bilan ishlamaydi
if a.casefold() == b.casefold():                    # ✅

if ism == "O'zbekiston":                            # ❌ faqat bitta variant
if uz_normalize(ism) == uz_normalize("O'zbekiston"):  # ✅

maydon = matn.upper()[:10]                          # ❌ ß → SS, uzunlik oshadi
maydon = matn[:10].upper()                          # ⚠️ hali ham muammo bo'lishi mumkin

if kirish.strip() == "ha":                          # ❌ ​ qoladi
if tozala(kirish) == "ha":                          # ✅ (4.4-dars)

# Bir foydalanuvchi NFC, boshqasi NFD saqlaydi → takrorlar
# Yechim: yozishdan oldin DOIM normalizatsiya

import unicodedata as ud

1.  len("café")           # NFC
2.  len("cafe\u0301")     # NFD: e + birikuvchi urg'u
3.  "caf\u00e9" == "cafe\u0301"     # NFC va NFD
4.  ud.normalize("NFC", "café") == "café"
5.  len("👨‍👩‍👧‍👦")
6.  len("🇺🇿")
7.  "ß".upper()
8.  len("ß".upper())
9.  ud.normalize("NFKC", "½")
10. ud.category("😊")
11. ord("а") == ord("a")
12. "İ".lower() == "i"

"Salom", "Салом", "Oʻzbekiston", "🇺🇿", "½①㎏"

kiril_lotin("Ўзбекистон")       # 'Oʻzbekiston'
lotin_kiril("Oʻzbekiston")      # 'Ўзбекистон'