saint/border-red-post-nalati
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat: persist isolated vector snapshots and exact retrieval

Browse Source
This commit is contained in:
Kronecker
2026-10-16 09:52:02 +08:00
parent 1c07d9ed16
commit 5e5cba2698
24 changed files with 1729 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/models/__init__.py
+12
View File
@@ -3,13 +3,22 @@ from app.models.catalog import AuditEvent, Document, Source
from app.models.indexing import Chunk, IndexPublication, IndexState, IndexVersion
from app.models.knowledge import KnowledgeItem
from app.models.retrieval import RetrievalRun
from app.models.vectors import (
ChunkVector,
EmbeddingSpace,
VectorPublication,
VectorState,
VectorVersion,
)
__all__ = [
"AuditEvent",
"AuthRateLimit",
"AuthSession",
"Chunk",
"ChunkVector",
"Document",
"EmbeddingSpace",
"IndexPublication",
"IndexState",
"IndexVersion",
@@ -17,4 +26,7 @@ __all__ = [
"RetrievalRun",
"Source",
"User",
"VectorPublication",
"VectorState",
"VectorVersion",
]
backend/app/models/indexing.py
+1
View File
@@ -50,6 +50,7 @@ class Chunk(Base):
__tablename__ = "chunks"
__table_args__ = (
UniqueConstraint("index_version_id", "knowledge_id", "ordinal", name="uq_chunk_position"),
UniqueConstraint("id", "index_version_id", name="uq_chunk_text_scope"),
CheckConstraint("start_offset >= 0 AND end_offset > start_offset", name="ck_chunk_offsets"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
backend/app/models/vectors.py
+112
View File
@@ -0,0 +1,112 @@
"""Versioned vector spaces, complete snapshots and independent publication history."""
from datetime import datetime
from pgvector.sqlalchemy import Vector
from sqlalchemy import (
JSON,
CheckConstraint,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Integer,
String,
UniqueConstraint,
func,
)
from sqlalchemy.dialects.postgresql.base import ischema_names
from sqlalchemy.orm import Mapped, mapped_column
from app.db.session import Base
# PostgreSQL reflects a qualified name when public is intentionally absent from search_path.
# Retain real type comparison during Alembic inspection in isolated schemas.
ischema_names.setdefault("public.vector", Vector)
class EmbeddingSpace(Base):
__tablename__ = "embedding_spaces"
__table_args__ = (
UniqueConstraint("id", "dimension", name="uq_embedding_space_dimension"),
CheckConstraint("dimension BETWEEN 1 AND 2000", name="ck_embedding_space_dimension"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
dimension: Mapped[int] = mapped_column(Integer, nullable=False)
spec: Mapped[dict] = mapped_column(JSON, nullable=False)
class VectorVersion(Base):
__tablename__ = "vector_versions"
__table_args__ = (
ForeignKeyConstraint(
["space_id", "dimension"],
["embedding_spaces.id", "embedding_spaces.dimension"],
name="fk_vector_version_space",
),
UniqueConstraint(
"id", "text_index_version_id", "space_id", "dimension", name="uq_vector_version_scope"
),
CheckConstraint(
"status IN ('ready', 'published', 'superseded')", name="ck_vector_version_status"
),
CheckConstraint("chunk_count BETWEEN 0 AND 2000", name="ck_vector_version_count"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
text_index_version_id: Mapped[str] = mapped_column(ForeignKey("index_versions.id"))
space_id: Mapped[str] = mapped_column(String(64), nullable=False)
dimension: Mapped[int] = mapped_column(Integer, nullable=False)
chunk_count: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="ready")
created_by: Mapped[int] = mapped_column(ForeignKey("users.id"))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class ChunkVector(Base):
__tablename__ = "chunk_vectors"
__table_args__ = (
ForeignKeyConstraint(
["version_id", "text_index_version_id", "space_id", "dimension"],
[
"vector_versions.id",
"vector_versions.text_index_version_id",
"vector_versions.space_id",
"vector_versions.dimension",
],
name="fk_chunk_vector_scope",
),
ForeignKeyConstraint(
["chunk_id", "text_index_version_id"],
["chunks.id", "chunks.index_version_id"],
name="fk_chunk_vector_text",
),
CheckConstraint("public.vector_dims(embedding) = dimension", name="ck_vector_dimension"),
CheckConstraint(
"public.vector_norm(embedding) BETWEEN 0.9999 AND 1.0001", name="ck_vector_unit_norm"
),
)
version_id: Mapped[str] = mapped_column(String(36), primary_key=True)
chunk_id: Mapped[str] = mapped_column(String(36), primary_key=True)
text_index_version_id: Mapped[str] = mapped_column(String(36), nullable=False)
space_id: Mapped[str] = mapped_column(String(64), nullable=False)
dimension: Mapped[int] = mapped_column(Integer, nullable=False)
embedding = mapped_column(Vector(), nullable=False)
class VectorState(Base):
__tablename__ = "vector_state"
__table_args__ = (CheckConstraint("id = 1", name="ck_vector_state_singleton"),)
id: Mapped[int] = mapped_column(primary_key=True)
active_version_id: Mapped[str | None] = mapped_column(ForeignKey("vector_versions.id"))
generation: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
class VectorPublication(Base):
__tablename__ = "vector_publications"
generation: Mapped[int] = mapped_column(primary_key=True)
previous_version_id: Mapped[str | None] = mapped_column(ForeignKey("vector_versions.id"))
version_id: Mapped[str] = mapped_column(ForeignKey("vector_versions.id"), nullable=False)
actor_id: Mapped[int] = mapped_column(ForeignKey("users.id"), nullable=False)
published_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)