Recherche privée dans vos connaissances
Ce projet construit la partie recherche d’une application RAG sur votre propre machine. Il ingère dix articles de support dans une base en mode enregistrement (chaque article reçoit son embedding et est indexé pour la recherche par mots-clés dans la même écriture), lance une recherche par mots-clés et une recherche hybride pour une question de client, peut reclasser les passages avec un cross-encoder, et affiche le prompt que votre application enverrait à son LLM. Rien n’appelle de LLM et, avec l’embedder par défaut, rien ne quitte la machine.
Prérequis
- Erlang/OTP 26 ou plus récent (cette exécution a utilisé OTP 29) et rebar3.
- Un compilateur C et CMake pour les dépendances natives (la première compilation prend quelques minutes).
- Pour
EMBEDDER=ollama: Ollama sur localhost, avecollama pull nomic-embed-text. - Pour
RERANK=1: un virtualenv Python avectransformersettorch(le modèle est téléchargé à la première utilisation).
Exécuter
./run.sh # offline, deterministic
EMBEDDER=ollama ./run.sh # real embeddings from Ollama
RERANK=1 RERANK_VENV=/path/to/venv ./run.sh
run.sh compile le projet, supprime ./data, puis exécute
private_rag:main/0 une fois.
Ce que montre l’exemple
barrel:open/2avec une politiqueembedding: lesfieldsreçoivent un embedding, lesmetadata_fieldssont conservés avec chaque vecteur pour filtrer.barrel:put_docs/2écrit l’article et son vecteur ensemble.barrel:search_bm25/3pour les mots-clés, ethybrid_top_ken BQL (vecteur et BM25, fusionnés par rang) filtré surarea.barrel_rerank:rerank/3réordonne les passages selon un score de cross-encoder (logits bruts, plus haut est meilleur).- Le prompt : une consigne, les meilleurs passages avec leurs identifiants, la question.
À propos des embedders
EMBEDDER=hash (par défaut) utilise hash_embed, un fournisseur
déterministe écrit pour cet exemple : il hache les mots sur 64
dimensions. Il trouve les passages qui partagent des mots avec la
question et ne comprend pas le sens, ses scores ne disent donc rien de
la qualité de la recherche. Il est là pour que l’exécution n’ait besoin
d’aucun modèle et donne la même sortie à chaque fois. EMBEDDER=ollama
utilise nomic-embed-text (768 dimensions) pour une vraie recherche
sémantique.
Sortie attendue
expected.txt provient de l’exécution par défaut, EMBEDDER=hash sans
reclassement : le nombre d’articles, les résultats par mots-clés et
hybrides avec leurs scores, et le prompt. Avec Ollama ou le
reclassement, l’ordre et les scores changent. Les deux options ont été
exécutées une fois à la main (Ollama avec nomic-embed-text, le
reclassement avec cross-encoder/ms-marco-MiniLM-L-6-v2) ; elles ne font
pas partie de la vérification automatique, qui doit tourner hors ligne.
Télécharger Le dossier sur GitHub, avec tous les fichiers ci-dessous.
Code source
%% Private knowledge search: ingest support articles in record mode,
%% run hybrid search for a question, optionally rerank, and print
%% the prompt an application would send to its LLM.
-module(private_rag).
-export([main/0]).
-define(QUESTION, <<"A customer was charged twice this month. "
"How do I refund one of the payments?">>).
main() ->
try run() of
ok -> halt(0)
catch C:R:S ->
io:format(standard_error, "~p:~p~n~p~n", [C, R, S]),
halt(1)
end.
run() ->
{ok, _} = application:ensure_all_started(barrel),
{Embedder, Dim} = embedder(os:getenv("EMBEDDER", "hash")),
%% Record mode: title and body are embedded in the same write,
%% and indexed for BM25 keyword search.
Policy = #{fields => [<<"title">>, <<"body">>], mode => sync,
embedder => Embedder, dimensions => Dim,
metadata_fields => [<<"area">>, <<"title">>]},
{ok, Db} = barrel:open(<<"kb">>, #{
embedding => Policy,
vectordb => #{db_path => filename:absname("data/vec/kb")}}),
{ok, Info} = barrel:embedder_info(Db),
io:format("embedder: ~p ~s, ~b dimensions~n",
[maps:get(provider, Info), maps:get(model, Info),
maps:get(dimensions, Info)]),
%% Ingest.
Path = filename:join(code:priv_dir(private_rag), "articles.eterm"),
{ok, Articles} = file:consult(Path),
Docs = [#{<<"id">> => Id, <<"area">> => Area,
<<"title">> => T, <<"body">> => B}
|| {Id, Area, T, B} <- Articles],
Written = [ok || {ok, _} <- barrel:put_docs(Db, Docs)],
io:format("ingested ~b articles~n", [length(Written)]),
%% Keyword only, then hybrid (vector + BM25, fused by rank).
io:format("~n~s", [wrap(<<"question: ", ?QUESTION/binary>>)]),
{ok, Kw} = barrel:search_bm25(Db, <<"refund charged twice">>,
#{k => 3}),
io:format("~nbm25 \"refund charged twice\":~n"),
[io:format(" ~.4f ~s~n", [S, Id]) || {Id, S} <- Kw],
{ok, Rows0, _} = barrel:query(Db,
<<"SELECT h.id, h._score, title, body "
"FROM hybrid_top_k($q, k => 4) AS h "
"WHERE h.area = 'billing'">>,
#{params => #{<<"q">> => ?QUESTION}}),
%% Best score first; equal scores by id, so the output is stable.
Rows = lists:sort(fun(A, B) -> key(A) =< key(B) end, Rows0),
io:format("~nhybrid, billing articles only:~n"),
[io:format(" ~.4f ~s~n", [maps:get(<<"_score">>, R),
maps:get(<<"id">>, R)])
|| R <- Rows],
Context = rerank(Rows, os:getenv("RERANK", "0")),
io:format("~n--- prompt (printed, not sent) ---~n~s",
[prompt(lists:sublist(Context, 3))]),
ok = barrel:close(Db),
ok.
key(#{<<"_score">> := S, <<"id">> := Id}) -> {-S, Id}.
%% EMBEDDER=hash is offline and deterministic: it matches shared
%% words, it does not understand meaning. EMBEDDER=ollama needs
%% `ollama pull nomic-embed-text` on localhost.
embedder("hash") ->
{{hash_embed, #{}}, 64};
embedder("ollama") ->
{{ollama, #{url => <<"http://localhost:11434">>,
model => <<"nomic-embed-text">>}}, 768}.
%% RERANK=1 reorders the passages with a cross-encoder (Python,
%% transformers and torch in RERANK_VENV).
rerank(Rows, "1") ->
ok = application:load(barrel_rerank),
ok = application:set_env(barrel_rerank, venv_path,
os:getenv("RERANK_VENV")),
{ok, _} = application:ensure_all_started(barrel_rerank),
{ok, Pid} = barrel_rerank:start_link(#{}),
Texts = [maps:get(<<"body">>, R) || R <- Rows],
{ok, Ranked} = barrel_rerank:rerank(Pid, ?QUESTION, Texts),
ok = barrel_rerank:stop(Pid),
io:format("~nreranked:~n"),
[io:format(" ~.3f ~s~n", [S, maps:get(<<"id">>,
lists:nth(I + 1, Rows))])
|| {I, S} <- Ranked],
[lists:nth(I + 1, Rows) || {I, _} <- Ranked];
rerank(Rows, _) ->
io:format("~nrerank: skipped (set RERANK=1)~n"),
Rows.
prompt(Passages) ->
Head = <<"Answer the question using only the support articles "
"below. Cite the article id for each step. If the "
"articles do not cover it, say so.">>,
[wrap(Head),
[[$\n, $[, Id, "] ", T, $\n, wrap(B)]
|| #{<<"id">> := Id, <<"title">> := T, <<"body">> := B}
<- Passages],
$\n, wrap(<<"Question: ", ?QUESTION/binary>>)].
%% Word-wrap to lines of at most 64 characters.
wrap(Text) ->
Words = string:lexemes(Text, " "),
{Lines, Last} = lists:foldl(
fun(W, {Acc, <<>>}) -> {Acc, W};
(W, {Acc, Cur}) ->
case string:length(Cur) + 1 + string:length(W) of
L when L > 64 -> {[Cur | Acc], W};
_ -> {Acc, <<Cur/binary, " ", W/binary>>}
end
end, {[], <<>>}, Words),
[[L, $\n] || L <- lists:reverse([Last | Lines])]. %% A deterministic, offline embedding provider for examples and tests:
%% 64-dimension feature hashing of the words. It matches shared words,
%% it does not understand meaning. Use a real model in production.
-module(hash_embed).
-behaviour(barrel_embed_provider).
-export([name/0, dimension/1, embed/2, embed_batch/2,
model_info/1]).
-define(DIM, 64).
name() -> hash_embed.
dimension(_Config) -> ?DIM.
model_info(_Config) -> #{model => <<"hash-64">>}.
embed(Text, _Config) -> {ok, vec(Text)}.
embed_batch(Texts, _Config) -> {ok, [vec(T) || T <- Texts]}.
vec(Text) ->
Words = string:lexemes(string:lowercase(Text),
" \n\t.,;:!?()'\""),
Zero = erlang:make_tuple(?DIM, 0.0),
Counts = lists:foldl(
fun(W, Acc) ->
I = erlang:phash2(W, ?DIM) + 1,
setelement(I, Acc, element(I, Acc) + 1.0)
end, Zero, Words),
L = tuple_to_list(Counts),
case math:sqrt(lists:sum([X * X || X <- L])) of
+0.0 -> L;
N -> [X / N || X <- L]
end. %% Support articles for a fictional invoicing app.
%% {Id, Area, Title, Body}.
{<<"kb:refund">>, <<"billing">>,
<<"Refund a payment">>,
<<"Open the invoice, choose Payments, then Refund next to the "
"payment. A full or partial refund goes back to the card used. "
"Card refunds take 5 to 10 business days to appear.">>}.
{<<"kb:double-charge">>, <<"billing">>,
<<"A customer was charged twice">>,
<<"A double charge usually comes from a retried checkout. Check "
"Payments on the invoice: if two payments share the same "
"amount and day, refund the second one. Pending duplicates "
"drop off within 3 days without a refund.">>}.
{<<"kb:export">>, <<"billing">>,
<<"Export invoices as CSV">>,
<<"Admins can export invoices from Reports, Export. Pick a date "
"range and the CSV format. Exports over 10000 rows are sent "
"by email.">>}.
{<<"kb:tax">>, <<"billing">>,
<<"Set tax rates per country">>,
<<"Go to Settings, Taxes and add a rate per country or region. "
"A rate applies to new invoices only; existing invoices keep "
"the rate they were issued with.">>}.
{<<"kb:password">>, <<"account">>,
<<"Reset a forgotten password">>,
<<"On the sign-in page choose Forgot password. The reset link "
"is valid for 30 minutes. Admins can also send a reset link "
"from Users.">>}.
{<<"kb:sso">>, <<"account">>,
<<"Configure single sign-on">>,
<<"SSO needs the Team plan. Upload your identity provider "
"metadata in Settings, Security, then map the email "
"attribute. Keep one admin with a password as a fallback.">>}.
{<<"kb:two-factor">>, <<"account">>,
<<"Turn on two-factor authentication">>,
<<"In Profile, Security, scan the QR code with an authenticator "
"app and save the recovery codes. Admins can require "
"two-factor for every user.">>}.
{<<"kb:webhooks">>, <<"integrations">>,
<<"Receive payment webhooks">>,
<<"Add an HTTPS endpoint in Settings, Webhooks. Each event is "
"signed; verify the signature header. Failed deliveries are "
"retried 5 times over 24 hours.">>}.
{<<"kb:api-keys">>, <<"integrations">>,
<<"Rotate API keys">>,
<<"Create the new key first, deploy it, then revoke the old key "
"in Settings, API. A revoked key stops working at once.">>}.
{<<"kb:plans">>, <<"account">>,
<<"Change your plan">>,
<<"Upgrades apply at once and are prorated. Downgrades apply at "
"the end of the billing period. SSO and audit logs need the "
"Team plan.">>}. {erl_opts, [debug_info]}.
%% barrel_rerank is only started when RERANK=1.
{deps, [{barrel, "~> 1.12"},
{barrel_rerank, "~> 1.0"}]}. #!/bin/sh
# Build, then run main/0 once on a fresh data directory.
# EMBEDDER=hash (default) | ollama RERANK=1 RERANK_VENV=<dir>
set -e
cd "$(dirname "$0")"
rebar3 compile >&2
rm -rf data
exec erl -noshell -config config/sys \
-pa _build/default/lib/*/ebin _build/default/checkouts/*/ebin \
-eval 'private_rag:main()' Sortie attendue
Le fichier expected.txt de l’exemple, issu d’une exécution réelle. Les révisions, identifiants, jetons et horodatages changent à chaque exécution ; scripts/verify-examples.sh les masque avant de comparer.
embedder: hash_embed hash-64, 64 dimensions
ingested 10 articles
question: A customer was charged twice this month. How do I
refund one of the payments?
bm25 "refund charged twice":
5.2942 kb:double-charge
2.2569 kb:refund
hybrid, billing articles only:
0.0164 kb:double-charge
0.0161 kb:refund
0.0152 kb:export
0.0152 kb:tax
rerank: skipped (set RERANK=1)
--- prompt (printed, not sent) ---
Answer the question using only the support articles below. Cite
the article id for each step. If the articles do not cover it,
say so.
[kb:double-charge] A customer was charged twice
A double charge usually comes from a retried checkout. Check
Payments on the invoice: if two payments share the same amount
and day, refund the second one. Pending duplicates drop off
within 3 days without a refund.
[kb:refund] Refund a payment
Open the invoice, choose Payments, then Refund next to the
payment. A full or partial refund goes back to the card used.
Card refunds take 5 to 10 business days to appear.
[kb:export] Export invoices as CSV
Admins can export invoices from Reports, Export. Pick a date
range and the CSV format. Exports over 10000 rows are sent by
email.
Question: A customer was charged twice this month. How do I
refund one of the payments?