26.5-dars: Scraping — Selenium / Playwright

Statik: server tayyor HTML yuboradi (requests ko'radi)
   → requests + BeautifulSoup yetarli

Dinamik: JS sahifani brauzerda quradi (requests ko'rmaydi)
   → real brauzer kerak (Selenium/Playwright)

# Playwright misol:
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto("https://example.com")
    kontent = page.content()      # JS ishlagach HTML
    browser.close()

page.goto(url)               # sahifaga o'tish
page.click("button.next")    # tugma bosish
page.fill("input#qidiruv", "python")  # forma to'ldirish
page.content()               # HTML (JS ishlagach)
page.screenshot(path="s.png")  # skrinshot

page.locator("button.next").click()          # topib bosish
page.locator(".narx").text_content()          # matn olish
page.locator(".mahsulot").all()               # barcha element
element.is_visible()                          # ko'rinadimi

page.wait_for_selector(".narx")       # element paydo bo'lguncha kut
page.wait_for_load_state("networkidle")  # tarmoq tinchlanguncha
# Selenium: WebDriverWait

browser = p.chromium.launch(headless=True)   # ekransiz (server)
browser = p.chromium.launch(headless=False)  # ko'rinadigan (debug)

# Playwright (zamonaviy):
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()
    page.goto("https://example.com")

    # amallar:
    page.click("button.next")             # bosish
    page.fill("input#q", "python")         # forma
    page.wait_for_selector(".narx")        # kutish
    narx = page.locator(".narx").text_content()
    kontent = page.content()               # HTML (BeautifulSoup bilan)

    browser.close()

# Selenium (eski):
# from selenium import webdriver
# driver = webdriver.Chrome()
# driver.get(url); driver.find_element(...).click()

# tanlov:
# statik → requests · dinamik → Playwright · API → requests (afzal)

Statik (requests) vs dinamik (JS — brauzer kerak) · goto/click/fill
locator (element) · wait (JS yuklanishi) · headless (ekransiz)
Playwright (yangi, tez) vs Selenium (eski) · brauzer — oxirgi chora (qimmat)

"""statik (requests ko'radi) vs dinamik (JS — brauzer kerak) — farqni ko'rsatish."""

import warnings
warnings.filterwarnings("ignore")


def requests_get(html_turi: str) -> str:
    """Soxta requests — faqat boshlang'ich HTML."""
    if html_turi == "statik":
        return "<div class='narx'>100</div>"     # ma'lumot bor
    return "<div id='app'></div>"                 # bo'sh (JS to'ldiradi)


def brauzer_get(html_turi: str) -> str:
    """Soxta brauzer — JS ishlagach HTML."""
    if html_turi == "statik":
        return "<div class='narx'>100</div>"
    return "<div id='app'><div class='narx'>100</div></div>"  # JS to'ldirdi


def main() -> None:
    print("=== 1. Statik sahifa (requests yetarli) ===")
    print(f"  requests: {requests_get('statik')}")
    print(f"  narx ko'rinadi: {'narx' in requests_get('statik')}")

    print("\n=== 2. Dinamik sahifa (requests bo'sh) ===")
    print(f"  requests: {requests_get('dinamik')}")
    print(f"  narx ko'rinadi: {'narx' in requests_get('dinamik')}")

    print("\n=== 3. Dinamik — brauzer kerak ===")
    print(f"  brauzer: {brauzer_get('dinamik')}")
    print(f"  narx ko'rinadi: {'narx' in brauzer_get('dinamik')}")

    print("\n=== 4. Tanlov ===")
    print("  statik → requests (tez)")
    print("  dinamik → brauzer (Selenium/Playwright)")
    print("  ⭐ statik vs dinamik — to'g'ri vosita tanlash")


if __name__ == "__main__":
    main()

=== 1. Statik sahifa (requests yetarli) ===
  requests: <div class='narx'>100</div>
  narx ko'rinadi: True

=== 2. Dinamik sahifa (requests bo'sh) ===
  requests: <div id='app'></div>
  narx ko'rinadi: False

=== 3. Dinamik — brauzer kerak ===
  brauzer: <div id='app'><div class='narx'>100</div></div>
  narx ko'rinadi: True

=== 4. Tanlov ===
  statik → requests (tez)
  dinamik → brauzer (Selenium/Playwright)
  ⭐ statik vs dinamik — to'g'ri vosita tanlash

"""mock brauzer: goto/click/fill/content — brauzerni boshqarish naqshi."""

import warnings
warnings.filterwarnings("ignore")


class MockPage:
    """Soxta Playwright page (deterministik)."""

    def __init__(self) -> None:
        self.url = None
        self.harakatlar: list = []

    def goto(self, url: str) -> None:
        self.url = url
        self.harakatlar.append(f"goto: {url}")

    def click(self, selektor: str) -> None:
        self.harakatlar.append(f"click: {selektor}")

    def fill(self, selektor: str, matn: str) -> None:
        self.harakatlar.append(f"fill: {selektor}={matn}")

    def content(self) -> str:
        return "<div class='natija'>Qidiruv natijasi</div>"


def main() -> None:
    page = MockPage()

    print("=== 1. Sahifaga o'tish (goto) ===")
    page.goto("https://example.com/search")
    print(f"  URL: {page.url}")

    print("\n=== 2. Forma to'ldirish (fill) ===")
    page.fill("input#qidiruv", "python kursi")
    print(f"  harakatlar: {page.harakatlar[-1]}")

    print("\n=== 3. Tugma bosish (click) ===")
    page.click("button.qidir")
    print(f"  harakatlar: {page.harakatlar[-1]}")

    print("\n=== 4. Kontent olish (content) ===")
    html = page.content()
    print(f"  HTML: {html}")
    print(f"  barcha amallar: {len(page.harakatlar)}")
    print("  ⭐ brauzer amallari — goto/fill/click/content")


if __name__ == "__main__":
    main()

=== 1. Sahifaga o'tish (goto) ===
  URL: https://example.com/search

=== 2. Forma to'ldirish (fill) ===
  harakatlar: fill: input#qidiruv=python kursi

=== 3. Tugma bosish (click) ===
  harakatlar: click: button.qidir

=== 4. Kontent olish (content) ===
  HTML: <div class='natija'>Qidiruv natijasi</div>
  barcha amallar: 3
  ⭐ brauzer amallari — goto/fill/click/content

"""kutish (wait — JS yuklanishi); elementlarni topish (locator) — dinamik naqsh."""

import warnings
warnings.filterwarnings("ignore")


class MockPage:
    """JS yuklanishini simulyatsiya qiluvchi soxta page."""

    def __init__(self) -> None:
        self.js_yuklandi = False

    def goto(self, url: str) -> None:
        self.js_yuklandi = False     # boshda JS yo'q

    def wait_for_selector(self, selektor: str) -> bool:
        self.js_yuklandi = True      # kutgach JS yuklandi
        return True

    def get_narx(self) -> str:
        return "100" if self.js_yuklandi else None


def main() -> None:
    page = MockPage()
    page.goto("https://example.com")

    print("=== 1. Kutmasdan (JS hali yo'q) ===")
    print(f"  narx (kutmasdan): {page.get_narx()}")

    print("\n=== 2. Kutish (wait_for_selector) ===")
    page.wait_for_selector(".narx")
    print(f"  JS yuklandi: {page.js_yuklandi}")

    print("\n=== 3. Kutgach (narx paydo bo'ldi) ===")
    print(f"  narx (kutgach): {page.get_narx()}")

    print("\n=== 4. Nega kutish muhim ===")
    print("  JS ma'lumotni keyinroq yuklaydi")
    print("  kutmasdan — None (element hali yo'q)")
    print("  ⭐ wait — dinamik sahifa scraping kaliti")


if __name__ == "__main__":
    main()

=== 1. Kutmasdan (JS hali yo'q) ===
  narx (kutmasdan): None

=== 2. Kutish (wait_for_selector) ===
  JS yuklandi: True

=== 3. Kutgach (narx paydo bo'ldi) ===
  narx (kutgach): 100

=== 4. Nega kutish muhim ===
  JS ma'lumotni keyinroq yuklaydi
  kutmasdan — None (element hali yo'q)
  ⭐ wait — dinamik sahifa scraping kaliti

"""to'liq dinamik scraping (mock): brauzer → goto → wait → content → BeautifulSoup tahlil."""

import warnings
warnings.filterwarnings("ignore")

from bs4 import BeautifulSoup


class MockBrowser:
    """Soxta brauzer (JS ishlagach HTML qaytaradi)."""

    def scrape(self, url: str) -> str:
        # JS ishlagach to'liq HTML (real brauzer buni beradi)
        return """
        <div class="mahsulot"><span class="nom">Python</span><span class="narx">100</span></div>
        <div class="mahsulot"><span class="nom">Go</span><span class="narx">120</span></div>
        """


def main() -> None:
    print("=== 1. Brauzer bilan sahifa olish (headless) ===")
    browser = MockBrowser()
    html = browser.scrape("https://example.com/dinamik")
    print("  brauzer JS ishladi, HTML olindi")

    print("\n=== 2. BeautifulSoup bilan tahlil 26.4-bob ===")
    soup = BeautifulSoup(html, "html.parser")
    mahsulotlar = soup.find_all("div", class_="mahsulot")
    print(f"  mahsulotlar soni: {len(mahsulotlar)}")

    print("\n=== 3. Ma'lumot ajratish ===")
    for div in mahsulotlar:
        nom = div.find("span", class_="nom").text
        narx = div.find("span", class_="narx").text
        print(f"  {nom}: {narx}")

    print("\n=== 4. Oqim ===")
    print("  brauzer (JS) → HTML → BeautifulSoup → ma'lumot")
    print("  dinamik sahifa ham scraping qilindi")
    print("  ⭐ Playwright/Selenium + BeautifulSoup — dinamik scraping")


if __name__ == "__main__":
    main()

=== 1. Brauzer bilan sahifa olish (headless) ===
  brauzer JS ishladi, HTML olindi

=== 2. BeautifulSoup bilan tahlil 26.4-bob ===
  mahsulotlar soni: 2

=== 3. Ma'lumot ajratish ===
  Python: 100
  Go: 120

=== 4. Oqim ===
  brauzer (JS) → HTML → BeautifulSoup → ma'lumot
  dinamik sahifa ham scraping qilindi
  ⭐ Playwright/Selenium + BeautifulSoup — dinamik scraping

requests.get(dinamik_url)   # bo'sh HTML                # ⚠️
# Playwright/Selenium (brauzer)                          # ✅

page.locator(".narx").text_content()   # hali yo'q      # ⚠️
page.wait_for_selector(".narx"); ...                     # ✅

# Playwright statik sahifaga                             # ⚠️ sekin
requests.get(statik_url)   # tez                          # ✅

browser = launch(); ...   # yopilmaydi                   # ⚠️ xotira
with sync_playwright() as p: ...   # avtomatik yopadi     # ✅

# yangi loyihada Selenium                                # ⚠️ (murakkabroq)
# Playwright (avtomatik kutish, tez)                      # ✅

# dinamik sahifa → darrov brauzer                        # ⚠️
# API bormi? (tarmoq — 26.3)                              # ✅ afzal

launch(headless=False)   # serverda ekran yo'q           # ⚠️
launch(headless=True)                                     # ✅

1.  # statik sahifa nima?
2.  # dinamik sahifa nima?
3.  # Selenium/Playwright nima?
4.  # goto nima?
5.  # click/fill nima?
6.  # locator nima?
7.  # wait nega?
8.  # headless nima?
9.  # Selenium vs Playwright?
10. # brauzer requests'dan?
11. # qachon brauzer?
12. # brauzer qimmatmi?

1.  requests.get(dinamik_url)              # brauzer

2.  page.locator(".narx")   # kutmasdan    # wait

3.  # Playwright statik sahifaga           # requests

4.  browser=launch(); ...   # yopilmaydi   # with

5.  launch(headless=False)   # serverda    # True