Indexer

# indexer.py
import sqlite3, sqlite_vec, requests, struct

OLLAMA = "http://localhost:11434"
MODELE = "zylonai/multilingual-e5-small"
DIMENSION = 384

def vecteur(texte):
    r = requests.post(f"{OLLAMA}/api/embeddings",
                      json={"model": MODELE, "prompt": texte}, timeout=120)
    return r.json()["embedding"]

def encoder(v):
    return struct.pack(f"{len(v)}f", *v)

base = sqlite3.connect("corpus.db")
base.enable_load_extension(True)
sqlite_vec.load(base)

base.execute("CREATE TABLE IF NOT EXISTS passages(id INTEGER PRIMARY KEY, source TEXT, texte TEXT)")
base.execute(f"CREATE VIRTUAL TABLE IF NOT EXISTS vecteurs USING vec0(id INTEGER PRIMARY KEY, v FLOAT[{DIMENSION}])")

for doc in docs:                      # docs vient de decouper.py
    cur = base.execute("INSERT INTO passages(source, texte) VALUES (?, ?)",
                       (doc["source"], doc["texte"]))
    base.execute("INSERT INTO vecteurs(id, v) VALUES (?, ?)",
                 (cur.lastrowid, encoder(vecteur(doc["texte"]))))

base.commit()
print("index terminé")