Доавление векторного поиска и репликации

library_service/main.py

@@ -11,6 +11,7 @@ from alembic import command
 from alembic.config import Config
 from fastapi import FastAPI, Depends, Request, Response, status
 from fastapi.staticfiles import StaticFiles
+from ollama import Client, ResponseError
 from sqlmodel import Session
 
 from library_service.auth import run_seeds
@@ -21,6 +22,7 @@ from library_service.settings import (
     engine,
     get_app,
     get_logger,
+    OLLAMA_URL,
 )
 
 
@@ -51,6 +53,13 @@ async def lifespan(_):
     except Exception as e:
         logger.error(f"[-] Seeding failed: {e}")
 
+    logger.info("[+] Loading ollama models...")
+    ollama_client = Client(host=OLLAMA_URL)
+    try:
+        ollama_client.pull("mxbai-embed-large")
+        ollama_client.pull("llama3.2")
+    except ResponseError as e:
+        logger.error(f"[-] Failed to pull models {e}")
     asyncio.create_task(cleanup_task())
     logger.info("[+] Starting application...")
     yield  # Обработка запросов

library_service/models/db/book.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING, List
 
+from pgvector.sqlalchemy import Vector
 from sqlalchemy import Column, String
 from sqlmodel import Field, Relationship
 
@@ -25,6 +26,7 @@ class Book(BookBase, table=True):
         sa_column=Column(String, nullable=False, default="active"),
         description="Статус",
     )
+    embedding: list[float] | None = Field(sa_column=Column(Vector(1024)))
     authors: List["Author"] = Relationship(
         back_populates="books", link_model=AuthorBookLink
     )

library_service/routers/books.py

@@ -1,13 +1,21 @@
 """Модуль работы с книгами"""
 
+from pydantic import Field
+from typing_extensions import Annotated
+
+from sqlalchemy.orm import selectinload, defer
+
+from sqlalchemy import text, case, distinct
+
 from datetime import datetime, timezone
 from typing import List
 
 from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
+from ollama import Client
 from sqlmodel import Session, select, col, func
 
 from library_service.auth import RequireStaff
-from library_service.settings import get_session
+from library_service.settings import get_session, OLLAMA_URL
 from library_service.models.enums import BookStatus
 from library_service.models.db import (
     Author,
@@ -32,6 +40,7 @@ from library_service.models.dto.misc import (
 
 
 router = APIRouter(prefix="/books", tags=["books"])
+ollama_client = Client(host=OLLAMA_URL)
 
 
 def close_active_loan(session: Session, book_id: int) -> None:
@@ -47,72 +56,64 @@ def close_active_loan(session: Session, book_id: int) -> None:
         session.add(active_loan)
 
 
-@router.get(
-    "/filter",
-    response_model=BookFilteredList,
-    summary="Фильтрация книг",
-    description="Фильтрация списка книг по названию, авторам и жанрам с пагинацией",
-)
+from sqlalchemy import select, func, distinct, case, exists
+from sqlalchemy.orm import selectinload
+
+
+@router.get("/filter", response_model=BookFilteredList)
 def filter_books(
     session: Session = Depends(get_session),
     q: str | None = Query(None, max_length=50, description="Поиск"),
-    min_page_count: int | None = Query(
-        None, ge=0, description="Минимальное количество страниц"
-    ),
-    max_page_count: int | None = Query(
-        None, ge=0, description="Максимальное количество страниц"
-    ),
-    author_ids: List[int] | None = Query(None, gt=0, description="Список ID авторов"),
-    genre_ids: List[int] | None = Query(None, gt=0, description="Список ID жанров"),
-    page: int = Query(1, gt=0, description="Номер страницы"),
-    size: int = Query(20, gt=0, le=100, description="Количество элементов на странице"),
+    min_page_count: int | None = Query(None, ge=0),
+    max_page_count: int | None = Query(None, ge=0),
+    author_ids: List[Annotated[int, Field(gt=0)]] | None = Query(None),
+    genre_ids: List[Annotated[int, Field(gt=0)]] | None = Query(None),
+    page: int = Query(1, gt=0),
+    size: int = Query(20, gt=0, le=100),
 ):
-    """Возвращает отфильтрованный список книг с пагинацией"""
-    statement = select(Book).distinct()
-
-    if q:
-        statement = statement.where(
-            (col(Book.title).ilike(f"%{q}%")) | (col(Book.description).ilike(f"%{q}%"))
-        )
+    statement = select(Book).options(
+        selectinload(Book.authors), selectinload(Book.genres), defer(Book.embedding)
+    )
 
     if min_page_count:
         statement = statement.where(Book.page_count >= min_page_count)
-
     if max_page_count:
         statement = statement.where(Book.page_count <= max_page_count)
 
     if author_ids:
-        statement = statement.join(AuthorBookLink).where(
-            AuthorBookLink.author_id.in_(  # ty: ignore[unresolved-attribute, unresolved-reference]
-                author_ids
+        statement = statement.where(
+            exists().where(
+                AuthorBookLink.book_id == Book.id,
+                AuthorBookLink.author_id.in_(author_ids),
             )
         )
 
     if genre_ids:
-        statement = statement.join(GenreBookLink).where(
-            GenreBookLink.genre_id.in_(  # ty: ignore[unresolved-attribute, unresolved-reference]
-                genre_ids
+        for genre_id in genre_ids:
+            statement = statement.where(
+                exists().where(
+                    GenreBookLink.book_id == Book.id, GenreBookLink.genre_id == genre_id
+                )
             )
-        )
 
-    total_statement = select(func.count()).select_from(statement.subquery())
-    total = session.exec(total_statement).one()
+    count_statement = select(func.count()).select_from(statement.subquery())
+    total = session.scalar(count_statement)
+
+    if q:
+        emb = ollama_client.embeddings(model="mxbai-embed-large", prompt=q)["embedding"]
+        distance_col = Book.embedding.cosine_distance(emb)
+        statement = statement.where(Book.embedding.is_not(None))
+
+        keyword_match = case((Book.title.ilike(f"%{q}%"), 0), else_=1)
+        statement = statement.order_by(keyword_match, distance_col)
+    else:
+        statement = statement.order_by(Book.id)
 
     offset = (page - 1) * size
     statement = statement.offset(offset).limit(size)
-    results = session.exec(statement).all()
+    results = session.scalars(statement).unique().all()
 
-    books_with_data = []
-    for db_book in results:
-        books_with_data.append(
-            BookWithAuthorsAndGenres(
-                **db_book.model_dump(),
-                authors=[AuthorRead(**a.model_dump()) for a in db_book.authors],
-                genres=[GenreRead(**g.model_dump()) for g in db_book.genres],
-            )
-        )
-
-    return BookFilteredList(books=books_with_data, total=total)
+    return BookFilteredList(books=results, total=total)
 
 
 @router.post(
@@ -127,11 +128,13 @@ def create_book(
     session: Session = Depends(get_session),
 ):
     """Создает новую книгу в системе"""
-    db_book = Book(**book.model_dump())
+    full_text = book.title + " " + book.description
+    emb = ollama_client.embeddings(model="mxbai-embed-large", prompt=full_text)
+    db_book = Book(**book.model_dump(), embedding=emb["embedding"])
     session.add(db_book)
     session.commit()
     session.refresh(db_book)
-    return BookRead(**db_book.model_dump())
+    return BookRead(**db_book.model_dump(exclude={"embedding"}))
 
 
 @router.get(
@@ -144,7 +147,8 @@ def read_books(session: Session = Depends(get_session)):
     """Возвращает список всех книг"""
     books = session.exec(select(Book)).all()
     return BookList(
-        books=[BookRead(**book.model_dump()) for book in books], total=len(books)
+        books=[BookRead(**book.model_dump(exclude={"embedding"})) for book in books],
+        total=len(books),
     )
 
 
@@ -165,19 +169,19 @@ def get_book(
             status_code=status.HTTP_404_NOT_FOUND, detail="Book not found"
         )
 
-    authors = session.exec(
+    authors = session.scalars(
         select(Author).join(AuthorBookLink).where(AuthorBookLink.book_id == book_id)
     ).all()
 
     author_reads = [AuthorRead(**author.model_dump()) for author in authors]
 
-    genres = session.exec(
+    genres = session.scalars(
         select(Genre).join(GenreBookLink).where(GenreBookLink.book_id == book_id)
     ).all()
 
     genre_reads = [GenreRead(**genre.model_dump()) for genre in genres]
 
-    book_data = book.model_dump()
+    book_data = book.model_dump(exclude={"embedding"})
     book_data["authors"] = author_reads
     book_data["genres"] = genre_reads
 
@@ -186,7 +190,7 @@ def get_book(
 
 @router.put(
     "/{book_id}",
-    response_model=Book,
+    response_model=BookRead,
     summary="Обновить информацию о книге",
     description="Обновляет информацию о книге в системе",
 )
@@ -221,11 +225,19 @@ def update_book(
         if book_update.description is not None:
             db_book.description = book_update.description
 
+        full_text = (
+            (book_update.title or db_book.title)
+            + " "
+            + (book_update.description or db_book.description)
+        )
+        emb = ollama_client.embeddings(model="mxbai-embed-large", prompt=full_text)
+        db_book.embedding = emb["embedding"]
+
     session.add(db_book)
     session.commit()
     session.refresh(db_book)
 
-    return BookRead(**db_book.model_dump())
+    return BookRead(**db_book.model_dump(exclude={"embedding"}))
 
 
 @router.delete(
@@ -249,6 +261,7 @@ def delete_book(
         id=(book.id or 0),
         title=book.title,
         description=book.description,
+        page_count=book.page_count,
         status=book.status,
     )
     session.delete(book)

library_service/settings.py

@@ -95,7 +95,9 @@ USER = os.getenv("POSTGRES_USER")
 PASSWORD = os.getenv("POSTGRES_PASSWORD")
 DATABASE = os.getenv("POSTGRES_DB")
 
-if not all([HOST, PORT, USER, PASSWORD, DATABASE]):
+OLLAMA_URL = os.getenv("OLLAMA_URL")
+
+if not all([HOST, PORT, USER, PASSWORD, DATABASE, OLLAMA_URL]):
     raise ValueError("Missing required POSTGRES environment variables")
 
 POSTGRES_DATABASE_URL = f"postgresql://{USER}:{PASSWORD}@{HOST}:{PORT}/{DATABASE}"

library_service/static/page/create_book.js

@@ -242,7 +242,7 @@ $(document).ready(() => {
       return;
     }
 
-    if (!parseInt(pageCount)) {
+    if (!pageCount) {
       Utils.showToast("Введите количество страниц", "error");
       return;
     }
@@ -253,7 +253,7 @@ $(document).ready(() => {
       const bookPayload = {
         title: title,
         description: description || null,
-        page_count: pageCount ? parseInt(pageCount) : null,
+        page_count: pageCount,
       };
 
       const createdBook = await Api.post("/api/books/", bookPayload);

library_service/static/page/edit_book.js

@@ -331,7 +331,7 @@ $(document).ready(() => {
 
     const title = $titleInput.val().trim();
     const description = $descInput.val().trim();
-    const pages = $pagesInput.val();
+    const pages = parseInt($("#book-page-count").val()) || null;
     const status = $statusSelect.val();
 
     if (!title) {
@@ -343,7 +343,7 @@ $(document).ready(() => {
     if (title !== originalBook.title) payload.title = title;
     if (description !== (originalBook.description || ""))
       payload.description = description || null;
-    if (pageCount !== originalBook.page_count) payload.page_count = pages;
+    if (pages !== originalBook.page_count) payload.page_count = pages;
     if (status !== originalBook.status) payload.status = status;
 
     if (Object.keys(payload).length === 0) {

library_service/templates/api.html

@@ -168,8 +168,7 @@
             jsPlumb.ready(function () {
                 const instance = jsPlumb.getInstance({
                     Container: "erDiagram",
-                    Connector: ["Flowchart", { stub: 30, gap: 10, cornerRadius: 5, alwaysRespectStubs: true }],
-                    ConnectionOverlays: [["Arrow", { location: 1, width: 10, length: 10, foldback: 0.8 }]]
+                    Connector: ["Flowchart", { stub: 30, gap: 10, cornerRadius: 5, alwaysRespectStubs: true }]
                 });
 
                 const container = document.getElementById("erDiagram");

library_service/templates/edit_genre.html

@@ -120,7 +120,7 @@
                 </button>
                 <a
                     id="cancel-btn"
-                    href="/"
+                    href="/books"
                     class="flex-1 flex justify-center items-center px-6 py-3 bg-white border border-gray-300 text-gray-700 font-medium rounded-lg hover:bg-gray-50 transition text-center"
                 >
                     Отмена