Embedding va semantik qidiruv: matnni vektorga aylantirib, ma'no bo'yicha topish

// o'qlar: [ovqat, salomatlik, transport] — o'yinchoq vektorlar
const vazifalar = {
  "Non va sut olish": [0.9, 0.1, 0],
  "Shifokor qabuliga yozilish": [0.05, 0.95, 0.1],
  "Mashinani ko'rikdan o'tkazish": [0, 0.1, 0.95],
};
const savol = [0.1, 0.9, 0.05]; // "doktor"

const skalyar = (a, b) => a.reduce((s, x, i) => s + x * b[i], 0);
const uzunlik = (a) => Math.sqrt(skalyar(a, a));
const kosinus = (a, b) => skalyar(a, b) / (uzunlik(a) * uzunlik(b));

for (const [matn, v] of Object.entries(vazifalar)) {
  console.log(kosinus(savol, v).toFixed(3), matn);
}

const birlik = (a) => a.map((x) => x / uzunlik(a));
const s1 = birlik(savol);
const v1 = birlik(vazifalar["Shifokor qabuliga yozilish"]);
console.log("uzunlik:", uzunlik(s1).toFixed(4));
console.log("skalyar:", skalyar(s1, v1).toFixed(3));

0.219 Non va sut olish
0.997 Shifokor qabuliga yozilish
0.159 Mashinani ko'rikdan o'tkazish
uzunlik: 1.0000
skalyar: 0.997

export function normallashtir(v: number[]): number[] {
  const u = Math.hypot(...v);
  return u === 0 ? v : v.map((x) => x / u);
}

const taskType = tur === "hujjat"
  ? "RETRIEVAL_DOCUMENT" : "RETRIEVAL_QUERY";
const r = await this.#post(`${model}:batchEmbedContents`, {
  requests: matnlar.map((text) => ({
    model, content: { parts: [{ text }] }, taskType,
    outputDimensionality: this.#s.embeddingOlcham,
  })),
}, s);

import { embedMany, cosineSimilarity } from "ai";

const { embeddings, usage } = await embedMany({
  model, // soxta: 3 o'lchamli vektor qaytaradi
  values: [
    "Non va sut olish", "Shifokor qabuliga yozilish", "doktor",
  ],
});
console.log("chaqiruvlar:", chaqiruv, "| tokenlar:", usage.tokens);
const [non, shifokor, doktor] = embeddings;
console.log(cosineSimilarity(doktor, shifokor).toFixed(3));
console.log(cosineSimilarity(doktor, non).toFixed(3));

chaqiruvlar: 2 | tokenlar: 12
0.994
0.061

"doktor": embedding top3 = Shifokor qabuliga yozilish 0.612-bob | Tish shifokoriga borish 0.571-bob | Hisobot yozish #ish 0.497-bob  →  2/2;  so'z mosligi → 0/2 (hech narsa)
recall@3: embedding 11/11 (100 %), so'z mosligi (FTS'ga yaqin) 0/11 (0 %)

CREATE TEMP TABLE v20 (id int, matn text);
INSERT INTO v20 VALUES
 (1, 'Non va sut sotib olish'), (2, 'Hisobot yozish #ish'),
 (3, 'IELTS imtihoniga ro''yxatdan o''tish'),
 (4, 'Bozordan meva va sabzavot olish'),
 (5, 'Shifokor qabuliga yozilish'),
 (6, 'Mashinani texnik ko''rikdan o''tkazish'),
 (7, 'Onamga tug''ilgan kun sovg''asi tanlash'),
 (8, 'Kvartira ijara haqini to''lash'),
 (9, 'JavaScript kursining 26-qismini o''qish'),
 (10, 'Sport zaliga borish'), (11, 'Jamoa bilan haftalik yig''ilish'),
 (12, 'Pasportni almashtirish uchun hujjat topshirish'),
 (13, 'Kir yuvish mashinasini ta''mirlatish'),
 (14, 'Bolalarni bog''chaga olib borish'),
 (15, 'Elektr to''lovini amalga oshirish'),
 (16, 'Kitob o''qish: Atomic Habits'),
 (17, 'Velosiped g''ildiragini almashtirish'),
 (18, 'Do''stlar bilan osh tayyorlash'),
 (19, 'Rezyumeni yangilash va vakansiyalarga yuborish'),
 (20, 'Tish shifokoriga borish');
SELECT q AS savol,
  coalesce(string_agg(v.id::text, ',' ORDER BY v.id), '—') AS topildi
FROM unnest(ARRAY['oziq-ovqat xaridi', 'kommunal to''lovlar',
  'doktor', 'ish qidirish', 'ingliz tili sertifikati',
  'машинани кўрикдан ўтказиш', 'birthday gift for mom',
  'avtomobil']) WITH ORDINALITY AS s(q, n)
LEFT JOIN v20 v ON to_tsvector('simple', uz_normal(v.matn)) @@
  nullif(replace(plainto_tsquery('simple', uz_normal(s.q))::text,
    '&', '|'), '')::tsquery
GROUP BY s.n, s.q ORDER BY s.n;

CREATE TABLE
INSERT 0 20
           savol           | topildi 
---------------------------+---------
 oziq-ovqat xaridi         | —
 kommunal to'lovlar        | —
 doktor                    | —
 ish qidirish              | 2
 ingliz tili sertifikati   | —
 машинани кўрикдан ўтказиш | 6
 birthday gift for mom     | —
 avtomobil                 | —
(8 rows)

SET LOCAL hnsw.iterative_scan = relaxed_order;

CREATE TEMP TABLE vaz (id int, matn text);
CREATE TEMP TABLE emb (vazifa_id int, model text, matn_hash bytea);
INSERT INTO vaz VALUES (1, 'Non olish'), (2, 'Hisobot yozish'),
  (3, 'Dori olish'), (4, 'Sport zaliga borish');
-- 1: o'zgarmagan; 2: matni o'zgargan; 3: eski model; 4: hali yo'q
INSERT INTO emb VALUES
  (1, 'gemini-embedding-2', sha256(convert_to('Non olish', 'UTF8'))),
  (2, 'gemini-embedding-2', sha256(convert_to('Hisobot', 'UTF8'))),
  (3, 'eski-model', sha256(convert_to('Dori olish', 'UTF8')));
SELECT v.id, v.matn,
  CASE WHEN e.vazifa_id IS NULL THEN 'yangi'
       WHEN e.model <> 'gemini-embedding-2' THEN 'model almashgan'
       ELSE 'matn o''zgargan' END AS sabab
FROM vaz v LEFT JOIN emb e ON e.vazifa_id = v.id
WHERE e.vazifa_id IS NULL
   OR e.matn_hash <> sha256(convert_to(v.matn, 'UTF8'))
   OR e.model <> 'gemini-embedding-2'
ORDER BY v.id;

CREATE TABLE
CREATE TABLE
INSERT 0 4
INSERT 0 3
 id |        matn         |      sabab      
----+---------------------+-----------------
  2 | Hisobot yozish      | matn o'zgargan
  3 | Dori olish          | model almashgan
  4 | Sport zaliga borish | yangi
(3 rows)

-- 303-embeddinglar.sql — vazifa matnining embedding'i. Alohida
-- jadval: model yoki o'lcham almashsa — qayta to'ldiriladi;
-- matn_hash — matn o'zgarganini bilish. CREATE EXTENSION —
-- superuser/egasi huquqi: migratsiya roli bilan, API roli bilan EMAS.
CREATE EXTENSION IF NOT EXISTS vector;

CREATE TABLE vazifa_embeddinglar (
  vazifa_id        bigint      NOT NULL,
  egasi_id         bigint      NOT NULL,
  model            text        NOT NULL,
  matn_hash        bytea       NOT NULL, -- sha256(matn)
  embedding        vector(768) NOT NULL, -- normallashtirilgan
  yangilangan_vaqt timestamptz NOT NULL DEFAULT now(),
  CONSTRAINT vazifa_embeddinglar_pkey PRIMARY KEY (vazifa_id),
  CONSTRAINT vazifa_embeddinglar_vazifa_id_fkey
    FOREIGN KEY (vazifa_id)
    REFERENCES vazifalar (id) ON DELETE CASCADE,
  CONSTRAINT vazifa_embeddinglar_egasi_id_fkey
    FOREIGN KEY (egasi_id)
    REFERENCES foydalanuvchilar (id) ON DELETE CASCADE
);
-- HNSW — cosine masofa (<=>); vektorlar normallashtirilgan —
-- inner product (<#>) ham bir xil tartib beradi
CREATE INDEX vazifa_embeddinglar_hnsw ON vazifa_embeddinglar
  USING hnsw (embedding vector_cosine_ops);
CREATE INDEX vazifa_embeddinglar_egasi_id_idx
  ON vazifa_embeddinglar (egasi_id);

GRANT SELECT, INSERT, UPDATE, DELETE ON vazifa_embeddinglar
  TO vazifalar_ilova;
ALTER TABLE vazifa_embeddinglar ENABLE ROW LEVEL SECURITY;
CREATE POLICY vazifa_embeddinglar_egasi ON vazifa_embeddinglar
  TO vazifalar_ilova
  USING (egasi_id = joriy_foydalanuvchi())
  WITH CHECK (egasi_id = joriy_foydalanuvchi());

COMMENT ON TABLE vazifa_embeddinglar
  IS 'vazifa embeddinglari (pgvector, HNSW cosine) — qidiruv va RAG';

  experimental: { externalTables: true },
  tables: {
    external: [
      "public.vazifa_hodisalari",
      // vector(768) + HNSW — Prisma bu turni bilmaydi
      "public.vazifa_embeddinglar",
    ],
  },

// qidiruv.ts — semantik qidiruv va RAG uchun topish (retrieval).
//   embedding'lar — dangasa (lazy) yangilanadi: qidiruvdan oldin shu
//   foydalanuvchining yangi/o'zgargan (matn_hash) vazifalari paketlab
//   hisoblanadi. semantik — pgvector HNSW (cosine); gibrid — FTS
//   (`qidiruv` tsvector, uz_normal) + vektor, Reciprocal Rank Fusion
//   (RRF, k = 60).
import type { Pool, PoolClient } from "pg";
/** pgvector matn shakli `[0.1,0.2,…]` — JSON massivi bilan bir xil
 *  (pgvector npm paketi kerak emas) */
const vektorSql = (v: number[]): string => JSON.stringify(v);
import { egasiSifatida } from "../umumiy/rls.ts";
import type { LlmProvayder } from "./llm.ts";

export type Topilgan = {
  id: number; matn: string; bajarildi: boolean;
  muddat: string | null; ball: number;
};
export type QidiruvSozlamasi = {
  pool: Pool; provayder: LlmProvayder; model: string; tz: string;
  paket?: number;
};

/** bitta embedding so'rovida ko'pi bilan nechta matn
 *  (Gemini batchEmbedContents — 100) */
const PAKET = 100;
const RRF_K = 60;

export class Qidiruv {
  constructor(private readonly s: QidiruvSozlamasi) {}

  /** yangi va matni o'zgargan vazifalar uchun embedding (bir
   *  chaqiruvda ≤ 100 ta); qaytaradi — nechta hisoblandi */
  async yangila(
    egasi: number, signal?: AbortSignal,
  ): Promise<number> {
    const paket = this.s.paket ?? PAKET;
    let jami = 0;
    for (;;) {
      type Eski = { id: string; matn: string; h: Buffer };
      const eskilar = await egasiSifatida(this.s.pool, egasi,
        async (c) => (await c.query<Eski>(
          `SELECT v.id, v.matn,
                  sha256(convert_to(v.matn, 'UTF8')) AS h
             FROM vazifalar v
             LEFT JOIN vazifa_embeddinglar e ON e.vazifa_id = v.id
            WHERE v.egasi_id = $1 AND v.ochirilgan_vaqt IS NULL
              AND (e.vazifa_id IS NULL
                OR e.matn_hash <> sha256(convert_to(v.matn, 'UTF8'))
                OR e.model <> $2)
            ORDER BY v.id LIMIT $3`,
          [egasi, this.s.model, paket])).rows);
      if (eskilar.length === 0) return jami;
      const vektorlar = await this.s.provayder.embedding(
        eskilar.map((q) => q.matn), "hujjat", signal);
      await egasiSifatida(this.s.pool, egasi, async (c) => {
        await c.query(
          `INSERT INTO vazifa_embeddinglar
             (vazifa_id, egasi_id, model, matn_hash, embedding)
           SELECT * FROM unnest($1::bigint[], $2::bigint[],
             $3::text[], $4::bytea[], $5::vector[])
           ON CONFLICT (vazifa_id) DO UPDATE SET
             model = EXCLUDED.model,
             matn_hash = EXCLUDED.matn_hash,
             embedding = EXCLUDED.embedding,
             yangilangan_vaqt = now()`,
          [eskilar.map((q) => q.id), eskilar.map(() => egasi),
            eskilar.map(() => this.s.model), eskilar.map((q) => q.h),
            vektorlar.map((v) => vektorSql(v))]);
      });
      jami += eskilar.length;
      if (eskilar.length < paket) return jami;
    }
  }

  async #vektorbilan(
    c: PoolClient, egasi: number, v: number[], k: number,
  ): Promise<Topilgan[]> {
    // pgvector 0.8: filtr (egasi) HNSW natijasidan KEYIN qo'llanadi —
    // ef_search (40) ta yaqindan boshqalarniki chiqib ketsa, natija
    // kam bo'ladi. iterative_scan — kerakli son topilguncha indeksni
    // davom ettiradi
    await c.query("SET LOCAL hnsw.iterative_scan = relaxed_order");
    const { rows } = await c.query<Topilgan & { id: string }>(
      `SELECT v.id, v.matn, v.bajarildi,
              to_char(v.muddat AT TIME ZONE $4, 'YYYY-MM-DD HH24:MI')
                AS muddat,
              1 - (e.embedding <=> $2) AS ball
         FROM vazifa_embeddinglar e
         JOIN vazifalar v ON v.id = e.vazifa_id
        WHERE e.egasi_id = $1 AND v.ochirilgan_vaqt IS NULL
        ORDER BY e.embedding <=> $2 LIMIT $3`,
      [egasi, vektorSql(v), k, this.s.tz]);
    // float8 — pg o'zi number qiladi; bigint id — satr
    return rows.map((r) => ({ ...r, id: Number(r.id) }));
  }

  async semantik(
    egasi: number, savol: string, k: number, signal?: AbortSignal,
  ): Promise<Topilgan[]> {
    await this.yangila(egasi, signal);
    const [v] = await this.s.provayder.embedding(
      [savol], "sorov", signal);
    if (v === undefined) return [];
    return egasiSifatida(this.s.pool, egasi,
      (c) => this.#vektorbilan(c, egasi, v, k));
  }

  /** gibrid: FTS (so'z mosligi — nom, raqam, teg) + vektor (ma'no) →
   *  RRF: ball = Σ 1/(k + o'rin) */
  async gibrid(
    egasi: number, savol: string, k: number, signal?: AbortSignal,
  ): Promise<Topilgan[]> {
    await this.yangila(egasi, signal);
    const [v] = await this.s.provayder.embedding(
      [savol], "sorov", signal);
    return egasiSifatida(this.s.pool, egasi, async (c) => {
      const vektor = v === undefined
        ? [] : await this.#vektorbilan(c, egasi, v, k * 2);
      // OR semantikasi: plainto_tsquery so'zlarni AND bilan
      // bog'laydi — savolda ortiqcha so'z bo'lsa hech narsa
      // topilmaydi
      const { rows } = await c.query<Topilgan & { id: string }>(
        `WITH s AS (SELECT nullif(replace(plainto_tsquery('simple',
            uz_normal($2))::text, '&', '|'), '')::tsquery AS q)
         SELECT v.id, v.matn, v.bajarildi,
                to_char(v.muddat AT TIME ZONE $4,
                  'YYYY-MM-DD HH24:MI') AS muddat,
                ts_rank(v.qidiruv, s.q) AS ball
           FROM vazifalar v, s
          WHERE v.egasi_id = $1 AND v.ochirilgan_vaqt IS NULL
            AND s.q IS NOT NULL AND v.qidiruv @@ s.q
          ORDER BY ball DESC, v.id LIMIT $3`,
        [egasi, savol, k * 2, this.s.tz]);
      const fts = rows.map((r) => ({ ...r, id: Number(r.id) }));
      const ballar = new Map<number, Topilgan>();
      for (const royxat of [vektor, fts]) {
        royxat.forEach((t, i) => {
          const b = ballar.get(t.id) ?? { ...t, ball: 0 };
          b.ball += 1 / (RRF_K + i + 1);
          ballar.set(t.id, b);
        });
      }
      return [...ballar.values()]
        .sort((a, b) => b.ball - a.ball || a.id - b.id)
        .slice(0, k);
    });
  }
}

import {
  AiVazifaController, QidiruvController,
} from "./ai.controller.ts";
import { Qidiruv } from "./qidiruv.ts";

export type AiBogliqliklari = {
  pool: Pool; provayder: LlmProvayder; tz?: string;
  /** embedding jadvalidagi model (o'zgarsa — qayta hisoblanadi) */
  embeddingModel?: string;
};
export type AiXizmatlari = {
  provayder: LlmProvayder; qadamlar: QadamlarXizmati;
  qidiruv: Qidiruv; log: Log;
};

function xizmatlar(b: AiBogliqliklari, log: Log): AiXizmatlari {
  const tz = b.tz ?? "Asia/Tashkent";
  const qidiruv = new Qidiruv({
    pool: b.pool, provayder: b.provayder, tz,
    model: b.embeddingModel ?? `${b.provayder.nom}-embedding`,
  });
  return {
    provayder: b.provayder, log, qidiruv,
    qadamlar: new QadamlarXizmati({
      pool: b.pool, provayder: b.provayder, tz,
    }),
  };
}

    if (o !== undefined || configdan) {
      controllers.push(AiVazifaController, QidiruvController);
    }
    // … config rejimida:
          return xizmatlar({
            pool: s.pool, provayder, tz: c.APP_TZ,
            embeddingModel: c.AI_EMBEDDING_MODEL,
          }, log);

import {
  Controller, Get, HttpCode, Inject, Param, Post, Query, Req, Res,
} from "@nestjs/common";
import { z } from "zod";
import { tekshir } from "../umumiy/tekshir.ts";

const QidiruvSorovi = z.strictObject({
  q: z.string().trim().min(2).max(200),
  rejim: z.enum(["semantik", "gibrid"]).default("gibrid"),
  limit: z.coerce.number().int().min(1).max(20).default(10),
});

/** GET /v1/qidiruv?q=…&rejim=semantik|gibrid — ma'no bo'yicha qidiruv
 *  (embedding'lar shu so'rovda dangasa yangilanadi) */
@Controller({ path: "qidiruv", version: "1" })
@Cheklov(API_CHEKLOVI)
export class QidiruvController {
  constructor(@Inject(AI) private readonly ai: AiXizmatlari) {}

  @Get()
  async qidir(
    @Egasi() egasi: number, @Query() q: unknown,
    @Req() req: Request, @Res({ passthrough: true }) res: Response,
  ): Promise<object> {
    const s = tekshir(QidiruvSorovi, q, "query");
    const signal = uzilishSignali(req, res);
    try {
      const topilgan = s.rejim === "semantik"
        ? await this.ai.qidiruv.semantik(egasi, s.q, s.limit, signal)
        : await this.ai.qidiruv.gibrid(egasi, s.q, s.limit, signal);
      return {
        rejim: s.rejim,
        vazifalar: topilgan.map((t) => ({
          ...t, ball: Math.round(t.ball * 10_000) / 10_000,
        })),
      };
    } catch (x) {
      throw aiXatosi(x);
    }
  }
}

const baza = await authBaza("ai_qidiruv");
if (typeof baza === "string") {
  test.skip(`AI qidiruv va RAG (Postgres + pgvector) — ${baza}`,
    () => {});
} else {
  const pool = baza;
  afterAll(() => pool.end());
  beforeEach(() => tozala(pool));
  const VAZIFALAR = [
    "Non va sut sotib olish", "Hisobot yozish #ish",
    "IELTS imtihoniga tayyorlanish", "Bozorga borib meva olish",
    "Sut mahsulotlari narxini solishtirish",
  ];
  const ilova = async (p: SoxtaProvayder) => {
    const d = authBogliqliklari(pool);
    const app = await expressIlova({
      ombor: (e) => new PgOmbori(pool, e), auth: d,
      ai: { pool, provayder: p, embeddingModel: "soxta-emb" },
    });
    const aziz = await hisobOch(app, d, "aziz@example.com");
    const dil = await hisobOch(app, d, "dilnoza@example.com");
    const idlar: number[] = [];
    for (const m of VAZIFALAR) {
      const r = await request(app).post("/v1/vazifalar")
        .set("Cookie", aziz).send({ matn: m }).expect(201);
      idlar.push((r.body as { id: number }).id);
    }
    await request(app).post("/v1/vazifalar").set("Cookie", dil)
      .send({ matn: "Sut va qatiq olish (Dilnoza)" }).expect(201);
    return { app, aziz, dil, idlar };
  };
  type Javob = {
    rejim: string;
    vazifalar: { id: number; matn: string; ball: number }[];
  };

  test("#17 semantik: embedding'lar dangasa (bir paketda), takror "
    + "so'rovda qayta hisoblanmaydi, matn o'zgarsa — faqat o'sha; "
    + "begona vazifalar yo'q", async () => {
    const p = new SoxtaProvayder();
    const { app, aziz, idlar } = await ilova(p);
    const r = (await request(app)
      .get("/v1/qidiruv?q=sut olish&rejim=semantik&limit=3")
      .set("Cookie", aziz).expect(200)).body as Javob;
    assert.equal(r.vazifalar[0]?.matn, "Non va sut sotib olish");
    assert.ok(r.vazifalar.every((v) => !v.matn.includes("Dilnoza")));
    assert.equal(p.embeddingChaqiruvlar, 2); // 1 paket + savol
    await request(app).get("/v1/qidiruv?q=hisobot&rejim=semantik")
      .set("Cookie", aziz).expect(200);
    assert.equal(p.embeddingChaqiruvlar, 3); // faqat savol
    await request(app).patch(`/v1/vazifalar/${String(idlar[1])}`)
      .set("Cookie", aziz)
      .set("Content-Type", "application/merge-patch+json")
      .send({ matn: "Choraklik hisobot yozish #ish" }).expect(200);
    await request(app).get("/v1/qidiruv?q=hisobot&rejim=semantik")
      .set("Cookie", aziz).expect(200);
    assert.equal(p.embeddingChaqiruvlar, 5); // o'zgargan 1 + savol
    const { rows } = await pool.query<{ n: number }>(
      "SELECT count(*)::int AS n FROM vazifa_embeddinglar");
    assert.equal(rows[0]?.n, 5); // Dilnozaniki hali hisoblanmagan
    await request(app).get("/v1/qidiruv?q=a")
      .set("Cookie", aziz).expect(400);
    await request(app).get("/v1/qidiruv?q=sut").expect(401);
  });

  test("#17 gibrid: FTS (so'z) + vektor → RRF; o'chirilgan vazifa "
    + "chiqmaydi", async () => {
    const { app, aziz, idlar } = await ilova(new SoxtaProvayder());
    await request(app).delete(`/v1/vazifalar/${String(idlar[0])}`)
      .set("Cookie", aziz).expect(204);
    const r = (await request(app).get("/v1/qidiruv?q=sut")
      .set("Cookie", aziz).expect(200)).body as Javob;
    assert.equal(r.rejim, "gibrid");
    assert.equal(r.vazifalar[0]?.matn,
      "Sut mahsulotlari narxini solishtirish");
    assert.ok(!r.vazifalar.some((v) => v.id === idlar[0]));
  });
}

npm run tip
npm run lint
npm test

 ✓ |asosiy| test/ai-qidiruv.test.ts (2 tests | 1 skipped) 9ms
 ✓ |asosiy| test/ai.test.ts (4 tests | 1 skipped) 16ms

 Test Files  2 passed (2)
      Tests  4 passed | 2 skipped (6)

--- GET /v1/qidiruv?q=dasturlash darslari (gibrid; birinchi so'rov embedding'larni hisoblaydi)
{"rejim":"gibrid","vazifalar":[{"id":1,"matn":"JavaScript kursining 26-qismini boshlash #o'qish","bajarildi":false,"muddat":"2026-10-31 23:59","ball":0.0164},{"id":34,"matn":"SQL JOIN mashqlarini yechish #o'qish","bajarildi":true,"muddat":"2026-10-25 21:00","ball":0.0161},{"id":17,"matn":"Portfolio saytiga blog yozish #portfolio","bajarildi":false,"muddat":"2026-11-15 18:00","ball":0.0159}]}
HTTP 200, 1.581261 s