16.8-dars: Katta fayllarni oqim bilan o'qish

def oqi(yol):
    with open(yol, encoding="utf-8") as f:
        for qator in f:
            yield qator.rstrip("\n")

def filtrla(qatorlar, shart):
    for q in qatorlar:
        if shart(q):
            yield q

def ozgartir(qatorlar, f):
    for q in qatorlar:
        yield f(q)

# hech narsa yuklanmaydi — element bittalab oqadi:
natija = ozgartir(filtrla(oqi("katta.txt"), shart), qayta_ishla)

[json.loads(q) for q in f]     # ❌ barcha yozuvlar xotirada
(json.loads(q) for q in f)     # ✅ bittalab, doimiy xotira

jami = sum(json.loads(q)["ball"] for q in f)      # butun fayl xotirada emas

from itertools import batched
for bolak in batched(yozuvlar, 1000):     # 3.12+
    con.executemany("INSERT ...", bolak)  # har 1000 tani bir tranzaksiyada

import gzip
with gzip.open("log.jsonl.gz", "rt", encoding="utf-8") as f:
    for qator in f:
        ...

# generator quvuri — doimiy xotira
def oqi(yol):
    with open(yol, encoding="utf-8") as f:
        yield from f

jami = sum(json.loads(q)["ball"] for q in oqi("katta.jsonl"))   # oqimli agregatsiya

import gzip
with gzip.open("log.jsonl.gz", "rt", encoding="utf-8") as f:    # siqilgan oqim
    for qator in f: ...

from itertools import batched
for bolak in batched(manba, 1000):                              # bo'laklab
    con.executemany("INSERT ...", bolak)

from collections import Counter
sanoq = Counter(json.loads(q)["tur"] for q in oqi("katta.jsonl"))

hech qachon .read()/.readlines()/json.load katta faylda
fayl obyekti — iterator: for qator in f
generator quvuri: yield, dangasa
agregatsiya: sum/Counter/max — hammani yuklamasdan
bo'laklab: itertools.batched
katta ma'lumot: JSON Lines, oddiy JSON emas
siqilgan: gzip.open(..., "rt")
to'liq saralash kerak bo'lsa: baza yoki tashqi saralash

"""fayl obyekti iterator; generator quvuri (oqi→filtrla→ozgartir); dangasa hisoblash; tracemalloc bilan ro'yxat vs generator xotirasi."""

import io
import itertools
import json
import tracemalloc


def oqi(matnlar):
    for qator in matnlar:
        yield qator.rstrip("\n")


def filtrla(qatorlar, shart):
    for q in qatorlar:
        if shart(q):
            yield q


def ozgartir(qatorlar, funksiya):
    for q in qatorlar:
        yield funksiya(q)


def main() -> None:
    print("=== 1. Generator quvuri ===")
    data = io.StringIO("\n".join(f'{{"id": {i}, "ball": {i * 10}}}' for i in range(10)))
    quvur = ozgartir(
        filtrla(oqi(data), lambda q: json.loads(q)["ball"] >= 30),
        lambda q: json.loads(q)["id"],
    )
    print(f"  quvur turi: {type(quvur).__name__} (dangasa — hali hech narsa o'qilmadi)")
    print(f"  birinchi 3 natija: {list(itertools.islice(quvur, 3))}")

    print("\n=== 2. Dangasa hisoblash ===")
    hisoblangan = []

    def kuzat(x):
        hisoblangan.append(x)
        return x

    gen = (kuzat(i) for i in range(1000))
    uchta = list(itertools.islice(gen, 3))
    print(f"  1000 elementli generatordan 3 ta olindi: {uchta}")
    print(f"  aslida hisoblangan: {len(hisoblangan)} ta (hammasi emas!)")

    print("\n=== 3. ⚠️ Xotira: ro'yxat vs generator ===")
    tracemalloc.start()
    royxat = [i * i for i in range(200_000)]
    royxat_xotira = tracemalloc.get_traced_memory()[1]
    tracemalloc.stop()

    tracemalloc.start()
    generator = (i * i for i in range(200_000))
    generator_xotira = tracemalloc.get_traced_memory()[1]
    tracemalloc.stop()

    print(f"  ro'yxat elementlari: {len(royxat)}")
    print(f"  generator — hali hisoblanmagan (dangasa)")
    print(f"  ro'yxat xotirasi generatordan kamida 100 barobar katta: {royxat_xotira > generator_xotira * 100}")
    print(f"  ikkalasining yig'indisi bir xil: {sum(royxat) == sum(i * i for i in range(200_000))}")


if __name__ == "__main__":
    main()

=== 1. Generator quvuri ===
  quvur turi: generator (dangasa — hali hech narsa o'qilmadi)
  birinchi 3 natija: [3, 4, 5]

=== 2. Dangasa hisoblash ===
  1000 elementli generatordan 3 ta olindi: [0, 1, 2]
  aslida hisoblangan: 3 ta (hammasi emas!)

=== 3. ⚠️ Xotira: ro'yxat vs generator ===
  ro'yxat elementlari: 200000
  generator — hali hisoblanmagan (dangasa)
  ro'yxat xotirasi generatordan kamida 100 barobar katta: True
  ikkalasining yig'indisi bir xil: True

"""katta JSON Lines faylni to'liq yuklash vs oqim — tracemalloc bilan peak xotira; yugurib borayotgan yig'indi va Counter; itertools.batched bilan bo'laklash."""

import itertools
import json
import tempfile
import tracemalloc
from collections import Counter
from pathlib import Path


def yarat(yol: Path, n: int) -> None:
    with open(yol, "w", encoding="utf-8") as f:
        for i in range(n):
            f.write(json.dumps({"id": i, "shahar": ["Toshkent", "Samarqand", "Buxoro"][i % 3], "ball": i % 100}) + "\n")


def main() -> None:
    with tempfile.TemporaryDirectory() as katalog:
        yol = Path(katalog) / "katta.jsonl"
        yarat(yol, 50_000)

        print("=== 1. ⚠️ To'liq yuklash vs oqim ===")
        tracemalloc.start()
        yozuvlar = [json.loads(q) for q in open(yol, encoding="utf-8")]
        toliq_yigindi = sum(y["ball"] for y in yozuvlar)
        toliq_peak = tracemalloc.get_traced_memory()[1]
        tracemalloc.stop()

        tracemalloc.start()
        oqim_yigindi = sum(json.loads(q)["ball"] for q in open(yol, encoding="utf-8"))
        oqim_peak = tracemalloc.get_traced_memory()[1]
        tracemalloc.stop()

        print(f"  yig'indi bir xil: {toliq_yigindi == oqim_yigindi}")
        print(f"  to'liq yuklash peak xotirasi oqimdan kamida 10 barobar katta: {toliq_peak > oqim_peak * 10}")

        print("\n=== 2. Oqimli agregatsiya ===")
        jami = soni = 0
        eng_kop = 0
        for qator in open(yol, encoding="utf-8"):
            y = json.loads(qator)
            jami += y["ball"]
            soni += 1
            eng_kop = max(eng_kop, y["ball"])
        print(f"  yozuvlar: {soni}, o'rtacha ball: {jami / soni:.2f}, eng ko'p: {eng_kop}")

        print("\n=== 3. Counter bilan guruhlash ===")
        sanoq = Counter(json.loads(q)["shahar"] for q in open(yol, encoding="utf-8"))
        for shahar, soni in sorted(sanoq.items()):
            print(f"  {shahar:12} {soni}")

        print("\n=== 4. itertools.batched bilan bo'laklash ===")
        manba = (json.loads(q)["id"] for q in open(yol, encoding="utf-8"))
        bolaklar_soni = 0
        oxirgi_bolak = None
        for bolak in itertools.batched(manba, 10_000):
            bolaklar_soni += 1
            oxirgi_bolak = bolak
        print(f"  50000 yozuv 10000 lik bo'laklarda: {bolaklar_soni} bo'lak")
        print(f"  oxirgi bo'lak uzunligi: {len(oxirgi_bolak)}, faqat bitta bo'lak xotirada")


if __name__ == "__main__":
    main()

=== 1. ⚠️ To'liq yuklash vs oqim ===
  yig'indi bir xil: True
  to'liq yuklash peak xotirasi oqimdan kamida 10 barobar katta: True

=== 2. Oqimli agregatsiya ===
  yozuvlar: 50000, o'rtacha ball: 49.50, eng ko'p: 99

=== 3. Counter bilan guruhlash ===
  Buxoro       16666
  Samarqand    16667
  Toshkent     16667

=== 4. itertools.batched bilan bo'laklash ===
  50000 yozuv 10000 lik bo'laklarda: 5 bo'lak
  oxirgi bo'lak uzunligi: 10000, faqat bitta bo'lak xotirada

"""gzip oqimli o'qish; XML iterparse + clear; CSV oqimli; sqlite fetchmany va iterator — barcha formatlarda 'hammasini yuklamaslik' naqshi."""

import csv
import gzip
import io
import json
import sqlite3
import tempfile
import xml.etree.ElementTree as ET
from pathlib import Path


def main() -> None:
    with tempfile.TemporaryDirectory() as katalog:
        r = Path(katalog)

        print("=== 1. gzip — siqilgan oqim ===")
        gz = r / "log.jsonl.gz"
        with gzip.open(gz, "wt", encoding="utf-8") as f:
            for i in range(100):
                f.write(json.dumps({"id": i, "ball": i}) + "\n")
        jami = 0
        with gzip.open(gz, "rt", encoding="utf-8") as f:
            for qator in f:                                  # butun fayl ochilmaydi
                jami += json.loads(qator)["ball"]
        print(f"  siqilgan fayl hajmi: {gz.stat().st_size} bayt, oqimli yig'indi: {jami}")

        print("\n=== 2. XML iterparse ===")
        xml = "<lenta>" + "".join(f"<yozuv><ball>{i}</ball></yozuv>" for i in range(50)) + "</lenta>"
        jami = soni = 0
        for hodisa, elem in ET.iterparse(io.StringIO(xml), events=("end",)):
            if elem.tag == "yozuv":
                jami += int(elem.findtext("ball"))
                soni += 1
                elem.clear()                                 # ⭐ xotirani bo'shatish
        print(f"  {soni} yozuv, yig'indi: {jami} (daraxt xotirada to'planmadi)")

        print("\n=== 3. CSV oqimli ===")
        yol = r / "data.csv"
        with open(yol, "w", encoding="utf-8", newline="") as f:
            w = csv.writer(f)
            w.writerow(["id", "ball"])
            w.writerows([(i, i * 2) for i in range(30)])
        with open(yol, encoding="utf-8", newline="") as f:
            jami = sum(int(q["ball"]) for q in csv.DictReader(f))     # qatorma-qator
        print(f"  CSV oqimli yig'indi: {jami}")

        print("\n=== 4. SQLite fetchmany va iterator ===")
        con = sqlite3.connect(":memory:")
        con.execute("CREATE TABLE t(id INTEGER, ball INTEGER)")
        con.executemany("INSERT INTO t VALUES(?, ?)", [(i, i) for i in range(100)])
        cur = con.execute("SELECT ball FROM t ORDER BY id")
        birinchi_bolak = cur.fetchmany(10)
        print(f"  fetchmany(10): {len(birinchi_bolak)} qator, boshi: {birinchi_bolak[0][0]}")
        jami = sum(ball for (ball,) in con.execute("SELECT ball FROM t"))   # iterator
        print(f"  iterator bilan yig'indi: {jami}")
        # agregatsiyani SQL o'zi qiladi — eng oqimli usul:
        print(f"  SQL AVG (baza o'zi hisoblaydi): {con.execute('SELECT AVG(ball) FROM t').fetchone()[0]}")
        con.close()


if __name__ == "__main__":
    main()

=== 1. gzip — siqilgan oqim ===
  siqilgan fayl hajmi: 438 bayt, oqimli yig'indi: 4950

=== 2. XML iterparse ===
  50 yozuv, yig'indi: 1225 (daraxt xotirada to'planmadi)

=== 3. CSV oqimli ===
  CSV oqimli yig'indi: 870

=== 4. SQLite fetchmany va iterator ===
  fetchmany(10): 10 qator, boshi: 0
  iterator bilan yig'indi: 4950
  SQL AVG (baza o'zi hisoblaydi): 49.5

"""ETL quvuri: gzip JSONL o'qish (Extract) → filtrlash/tozalash generatori (Transform) → SQLite ga batched yuklash + oqimli statistika (Load); doimiy xotira."""

import gzip
import json
import sqlite3
import tempfile
from collections import Counter
from contextlib import closing
from itertools import batched
from pathlib import Path


def yarat_jurnal(yol: Path, n: int) -> None:
    darajalar = ["INFO", "INFO", "INFO", "WARNING", "ERROR"]
    with gzip.open(yol, "wt", encoding="utf-8") as f:
        for i in range(n):
            yozuv = {"vaqt": f"2026-09-17T{i % 24:02d}:00:00", "daraja": darajalar[i % 5],
                     "foydalanuvchi": f"user{i % 7}", "davomiylik_ms": (i % 50) * 10}
            if i % 500 == 499:
                f.write("buzuq qator emas json\n")             # ataylab buzuq qator
            else:
                f.write(json.dumps(yozuv) + "\n")


def oqi(yol: Path):
    """Extract: gzip ni oqimli o'qib, buzuq qatorlarga bardosh berib, yozuv beradi."""
    with gzip.open(yol, "rt", encoding="utf-8") as f:
        for raqam, qator in enumerate(f, 1):
            try:
                yield json.loads(qator)
            except json.JSONDecodeError:
                continue                                      # buzuq qatorni o'tkazib yuborish


def tozala(yozuvlar):
    """Transform: faqat WARNING/ERROR va kerakli maydonlar."""
    for y in yozuvlar:
        if y.get("daraja") in {"WARNING", "ERROR"}:
            yield (y["vaqt"], y["daraja"], y["foydalanuvchi"], y["davomiylik_ms"])


def main() -> None:
    with tempfile.TemporaryDirectory() as katalog:
        r = Path(katalog)
        jurnal = r / "server.jsonl.gz"
        yarat_jurnal(jurnal, 10_000)
        print(f"=== 1. Kirish: {jurnal.stat().st_size} bayt (siqilgan) ===")

        with closing(sqlite3.connect(r / "hodisalar.db")) as con:
            con.execute("CREATE TABLE hodisa(vaqt TEXT, daraja TEXT, foydalanuvchi TEXT, davomiylik_ms INTEGER)")

            # Load: batched INSERT + oqimli statistika (bir o'tishda)
            daraja_sanoq: Counter = Counter()
            foydalanuvchi_sanoq: Counter = Counter()
            jami_davomiylik = qatorlar = 0

            def kuzatib_otkaz(yozuvlar):
                nonlocal jami_davomiylik, qatorlar
                for vaqt, daraja, fydl, ms in yozuvlar:
                    daraja_sanoq[daraja] += 1
                    foydalanuvchi_sanoq[fydl] += 1
                    jami_davomiylik += ms
                    qatorlar += 1
                    yield (vaqt, daraja, fydl, ms)

            with con:
                for bolak in batched(kuzatib_otkaz(tozala(oqi(jurnal))), 1000):
                    con.executemany("INSERT INTO hodisa VALUES(?, ?, ?, ?)", bolak)

            print(f"\n=== 2. Yuklandi: {qatorlar} hodisa (WARNING/ERROR) ===")
            print(f"  bazadagi qatorlar: {con.execute('SELECT COUNT(*) FROM hodisa').fetchone()[0]}")

            print("\n=== 3. Statistika (oqim davomida yig'ilgan) ===")
            for daraja, soni in sorted(daraja_sanoq.items()):
                print(f"  {daraja:8} {soni}")
            print(f"  o'rtacha davomiylik: {jami_davomiylik / qatorlar:.1f} ms")
            print(f"  eng faol foydalanuvchi: {foydalanuvchi_sanoq.most_common(1)[0]}")

            print("\n=== 4. Bazadan agregatsiya (SQL) ===")
            for fydl, soni, ortacha in con.execute(
                    "SELECT foydalanuvchi, COUNT(*), AVG(davomiylik_ms) FROM hodisa "
                    "GROUP BY foydalanuvchi ORDER BY COUNT(*) DESC LIMIT 3"):
                print(f"  {fydl:8} {soni} hodisa, o'rtacha {ortacha:.0f} ms")
            print("\n  ⭐ butun quvur davomida fayl xotiraga to'liq yuklanmadi")


if __name__ == "__main__":
    main()

=== 1. Kirish: 43309 bayt (siqilgan) ===

=== 2. Yuklandi: 3980 hodisa (WARNING/ERROR) ===
  bazadagi qatorlar: 3980

=== 3. Statistika (oqim davomida yig'ilgan) ===
  ERROR    1980
  WARNING  2000
  o'rtacha davomiylik: 258.8 ms
  eng faol foydalanuvchi: ('user3', 569)

=== 4. Bazadan agregatsiya (SQL) ===
  user6    569 hodisa, o'rtacha 259 ms
  user4    569 hodisa, o'rtacha 258 ms
  user3    569 hodisa, o'rtacha 259 ms

  ⭐ butun quvur davomida fayl xotiraga to'liq yuklanmadi

data = open("katta.txt").read()                     # ❌
for qator in open("katta.txt", encoding="utf-8"):   # ✅
    ...

yozuvlar = [json.loads(q) for q in f]               # ❌ hammasi xotirada
jami = sum(json.loads(q)["x"] for q in f)           # ✅ oqimli

json.load(open("10gb.json"))                        # ❌
# ✅ JSON Lines ga o'ting yoki ijson (oqimli parser)

qatorlar = cur.fetchall()                           # ❌
for qator in con.execute("SELECT ..."):             # ✅
    ...

list(generator)[0]                                  # ❌ hammasini oladi
next(generator)                                     # ✅ yoki islice

for _, e in ET.iterparse(f): ...                    # ❌ daraxt to'planadi
    ...; e.clear()                                  # ✅

d = defaultdict(list)                               # ⚠️ hammasi xotirada
for q in f: d[q["shahar"]].append(q)
# ✅ bazaga yuklab SQL GROUP BY, yoki Counter bilan faqat sanoq

gzip.open("log.gz", "rt").read()                    # ❌
for qator in gzip.open("log.gz", "rt", encoding="utf-8"):   # ✅
    ...

import itertools, json, io
1.  g = (i * 2 for i in range(5)); print(next(g), next(g))
2.  print(list(itertools.islice((i for i in range(100)), 3)))
3.  print(list(itertools.batched(range(7), 3)))
4.  f = io.StringIO("a\nb\nc\n"); print(sum(1 for _ in f))
5.  f = io.StringIO("1\n2\n3\n"); print(sum(int(x) for x in f))
6.  g = (x for x in [1, 2, 3]); print(type(g).__name__)
7.  print(list(itertools.islice(itertools.count(), 4)))
8.  from collections import Counter; print(Counter("aabbbc").most_common(1))
9.  g = (i for i in range(3)); list(g); print(list(g))
10. print(next(iter(itertools.batched("abcdef", 2))))
11. f = io.StringIO('{"x":1}\n{"x":2}\n'); print(sum(json.loads(l)["x"] for l in f))
12. print(list(itertools.chain([1, 2], [3, 4])))

1.  def qatorlar_soni(yol):
        return len(open(yol).read().split("\n"))

2.  def jami_ball(yol):
        yozuvlar = [json.loads(q) for q in open(yol)]        # 5 GB fayl
        return sum(y["ball"] for y in yozuvlar)

3.  def xatolar(yol):
        return list(ET.parse(yol).iterfind(".//error"))      # katta XML

4.  def eksport(con, yol):
        qatorlar = con.execute("SELECT * FROM katta").fetchall()
        with open(yol, "w") as f:
            for q in qatorlar: f.write(str(q))

5.  def gz_oqi(yol):
        matn = gzip.open(yol, "rt").read()                   # 8 GB siqilgan
        return matn.count("ERROR")