Billion-scale vector search, built on Parquet.

import parqdb

session = parqdb.connect("./parqdb-data")
session.register_parquet(
    "documents",
    parqdb.datasets.uri("documents"),
)
documents = session.table("documents")

documents.create_index(
    "documents_embedding",
    column="embedding",
    key=["document_id"],
    config=parqdb.IVF(nlist=3),
)
documents.wait_for_index("documents_embedding")

hits = session.collect(
    documents.search([0.2, 0.0])
    .where("status = 'published'")
    .limit(10)
)

$ pip install "parqdb[publish]"

$ parqdb publish \
    --source documents.parquet \
    --key chunk_id \
    --vector-column embedding \
    --nlist 4096 \
    --encoding lvq8 \
    --destination s3://my-bucket/wiki/v1 \
    --public-url https://data.example.com/wiki/v1

# manifest.json is exposed only after every
# immutable object has been uploaded and verified.