Barrel
FR GitHub

Private knowledge search

This project builds the retrieval half of a RAG application on your own machine. It ingests ten support articles into a record-mode database (each article is embedded and indexed for keyword search in the same write), runs a keyword search and a hybrid search for a customer question, can rerank the passages with a cross-encoder, and prints the prompt your application would send to its LLM. Nothing calls an LLM and, with the default embedder, nothing leaves the machine.

Prerequisites

  • Erlang/OTP 26 or later (this run used OTP 29) and rebar3.
  • A C compiler and CMake for the native dependencies (first compile takes a few minutes).
  • For EMBEDDER=ollama: Ollama on localhost with ollama pull nomic-embed-text.
  • For RERANK=1: a Python virtualenv with transformers and torch (the model downloads on first use).

Run

./run.sh                       # offline, deterministic
EMBEDDER=ollama ./run.sh       # real embeddings from Ollama
RERANK=1 RERANK_VENV=/path/to/venv ./run.sh

run.sh compiles the project, deletes ./data, then runs private_rag:main/0 once.

What it shows

  • barrel:open/2 with an embedding policy: fields are embedded, metadata_fields are kept with each vector for filtering.
  • barrel:put_docs/2 writes the article and its vector together.
  • barrel:search_bm25/3 for keywords, and BQL hybrid_top_k (vector and BM25, fused by rank) filtered on area.
  • barrel_rerank:rerank/3 reorders the passages by a cross-encoder score (raw logits, higher is better).
  • The prompt: an instruction, the top passages with their ids, the question.

About the embedders

EMBEDDER=hash (the default) uses hash_embed, a deterministic provider written for this example: it hashes words into 64 dimensions. It finds passages that share words with the question and does not understand meaning, so its scores say nothing about search quality. It is there so the run needs no model and gives the same output every time. EMBEDDER=ollama uses nomic-embed-text (768 dimensions) for real semantic search.

Expected output

expected.txt comes from the default run, EMBEDDER=hash without rerank: the article count, the keyword and hybrid hits with their scores, and the prompt. With Ollama or rerank the ranking and scores differ. Both options were run once by hand (Ollama with nomic-embed-text, rerank with cross-encoder/ms-marco-MiniLM-L-6-v2); they are not part of the automatic check, which must run offline.

Download The folder on GitHub, with every file below.

Source

src/private_rag.erl
%% Private knowledge search: ingest support articles in record mode,
%% run hybrid search for a question, optionally rerank, and print
%% the prompt an application would send to its LLM.
-module(private_rag).
-export([main/0]).

-define(QUESTION, <<"A customer was charged twice this month. "
                    "How do I refund one of the payments?">>).

main() ->
    try run() of
        ok -> halt(0)
    catch C:R:S ->
        io:format(standard_error, "~p:~p~n~p~n", [C, R, S]),
        halt(1)
    end.

run() ->
    {ok, _} = application:ensure_all_started(barrel),
    {Embedder, Dim} = embedder(os:getenv("EMBEDDER", "hash")),

    %% Record mode: title and body are embedded in the same write,
    %% and indexed for BM25 keyword search.
    Policy = #{fields => [<<"title">>, <<"body">>], mode => sync,
               embedder => Embedder, dimensions => Dim,
               metadata_fields => [<<"area">>, <<"title">>]},
    {ok, Db} = barrel:open(<<"kb">>, #{
        embedding => Policy,
        vectordb => #{db_path => filename:absname("data/vec/kb")}}),
    {ok, Info} = barrel:embedder_info(Db),
    io:format("embedder: ~p ~s, ~b dimensions~n",
              [maps:get(provider, Info), maps:get(model, Info),
               maps:get(dimensions, Info)]),

    %% Ingest.
    Path = filename:join(code:priv_dir(private_rag), "articles.eterm"),
    {ok, Articles} = file:consult(Path),
    Docs = [#{<<"id">> => Id, <<"area">> => Area,
              <<"title">> => T, <<"body">> => B}
            || {Id, Area, T, B} <- Articles],
    Written = [ok || {ok, _} <- barrel:put_docs(Db, Docs)],
    io:format("ingested ~b articles~n", [length(Written)]),

    %% Keyword only, then hybrid (vector + BM25, fused by rank).
    io:format("~n~s", [wrap(<<"question: ", ?QUESTION/binary>>)]),
    {ok, Kw} = barrel:search_bm25(Db, <<"refund charged twice">>,
                                  #{k => 3}),
    io:format("~nbm25 \"refund charged twice\":~n"),
    [io:format("  ~.4f ~s~n", [S, Id]) || {Id, S} <- Kw],

    {ok, Rows0, _} = barrel:query(Db,
        <<"SELECT h.id, h._score, title, body "
          "FROM hybrid_top_k($q, k => 4) AS h "
          "WHERE h.area = 'billing'">>,
        #{params => #{<<"q">> => ?QUESTION}}),
    %% Best score first; equal scores by id, so the output is stable.
    Rows = lists:sort(fun(A, B) -> key(A) =< key(B) end, Rows0),
    io:format("~nhybrid, billing articles only:~n"),
    [io:format("  ~.4f ~s~n", [maps:get(<<"_score">>, R),
                               maps:get(<<"id">>, R)])
     || R <- Rows],

    Context = rerank(Rows, os:getenv("RERANK", "0")),
    io:format("~n--- prompt (printed, not sent) ---~n~s",
              [prompt(lists:sublist(Context, 3))]),
    ok = barrel:close(Db),
    ok.

key(#{<<"_score">> := S, <<"id">> := Id}) -> {-S, Id}.

%% EMBEDDER=hash is offline and deterministic: it matches shared
%% words, it does not understand meaning. EMBEDDER=ollama needs
%% `ollama pull nomic-embed-text` on localhost.
embedder("hash") ->
    {{hash_embed, #{}}, 64};
embedder("ollama") ->
    {{ollama, #{url => <<"http://localhost:11434">>,
                model => <<"nomic-embed-text">>}}, 768}.

%% RERANK=1 reorders the passages with a cross-encoder (Python,
%% transformers and torch in RERANK_VENV).
rerank(Rows, "1") ->
    ok = application:load(barrel_rerank),
    ok = application:set_env(barrel_rerank, venv_path,
                             os:getenv("RERANK_VENV")),
    {ok, _} = application:ensure_all_started(barrel_rerank),
    {ok, Pid} = barrel_rerank:start_link(#{}),
    Texts = [maps:get(<<"body">>, R) || R <- Rows],
    {ok, Ranked} = barrel_rerank:rerank(Pid, ?QUESTION, Texts),
    ok = barrel_rerank:stop(Pid),
    io:format("~nreranked:~n"),
    [io:format("  ~.3f ~s~n", [S, maps:get(<<"id">>,
                                           lists:nth(I + 1, Rows))])
     || {I, S} <- Ranked],
    [lists:nth(I + 1, Rows) || {I, _} <- Ranked];
rerank(Rows, _) ->
    io:format("~nrerank: skipped (set RERANK=1)~n"),
    Rows.

prompt(Passages) ->
    Head = <<"Answer the question using only the support articles "
             "below. Cite the article id for each step. If the "
             "articles do not cover it, say so.">>,
    [wrap(Head),
     [[$\n, $[, Id, "] ", T, $\n, wrap(B)]
      || #{<<"id">> := Id, <<"title">> := T, <<"body">> := B}
             <- Passages],
     $\n, wrap(<<"Question: ", ?QUESTION/binary>>)].

%% Word-wrap to lines of at most 64 characters.
wrap(Text) ->
    Words = string:lexemes(Text, " "),
    {Lines, Last} = lists:foldl(
        fun(W, {Acc, <<>>}) -> {Acc, W};
           (W, {Acc, Cur}) ->
                case string:length(Cur) + 1 + string:length(W) of
                    L when L > 64 -> {[Cur | Acc], W};
                    _ -> {Acc, <<Cur/binary, " ", W/binary>>}
                end
        end, {[], <<>>}, Words),
    [[L, $\n] || L <- lists:reverse([Last | Lines])].
src/hash_embed.erl
%% A deterministic, offline embedding provider for examples and tests:
%% 64-dimension feature hashing of the words. It matches shared words,
%% it does not understand meaning. Use a real model in production.
-module(hash_embed).
-behaviour(barrel_embed_provider).
-export([name/0, dimension/1, embed/2, embed_batch/2,
         model_info/1]).

-define(DIM, 64).

name() -> hash_embed.
dimension(_Config) -> ?DIM.
model_info(_Config) -> #{model => <<"hash-64">>}.

embed(Text, _Config) -> {ok, vec(Text)}.
embed_batch(Texts, _Config) -> {ok, [vec(T) || T <- Texts]}.

vec(Text) ->
    Words = string:lexemes(string:lowercase(Text),
                           " \n\t.,;:!?()'\""),
    Zero = erlang:make_tuple(?DIM, 0.0),
    Counts = lists:foldl(
        fun(W, Acc) ->
            I = erlang:phash2(W, ?DIM) + 1,
            setelement(I, Acc, element(I, Acc) + 1.0)
        end, Zero, Words),
    L = tuple_to_list(Counts),
    case math:sqrt(lists:sum([X * X || X <- L])) of
        +0.0 -> L;
        N -> [X / N || X <- L]
    end.
priv/articles.eterm
%% Support articles for a fictional invoicing app.
%% {Id, Area, Title, Body}.
{<<"kb:refund">>, <<"billing">>,
 <<"Refund a payment">>,
 <<"Open the invoice, choose Payments, then Refund next to the "
   "payment. A full or partial refund goes back to the card used. "
   "Card refunds take 5 to 10 business days to appear.">>}.
{<<"kb:double-charge">>, <<"billing">>,
 <<"A customer was charged twice">>,
 <<"A double charge usually comes from a retried checkout. Check "
   "Payments on the invoice: if two payments share the same "
   "amount and day, refund the second one. Pending duplicates "
   "drop off within 3 days without a refund.">>}.
{<<"kb:export">>, <<"billing">>,
 <<"Export invoices as CSV">>,
 <<"Admins can export invoices from Reports, Export. Pick a date "
   "range and the CSV format. Exports over 10000 rows are sent "
   "by email.">>}.
{<<"kb:tax">>, <<"billing">>,
 <<"Set tax rates per country">>,
 <<"Go to Settings, Taxes and add a rate per country or region. "
   "A rate applies to new invoices only; existing invoices keep "
   "the rate they were issued with.">>}.
{<<"kb:password">>, <<"account">>,
 <<"Reset a forgotten password">>,
 <<"On the sign-in page choose Forgot password. The reset link "
   "is valid for 30 minutes. Admins can also send a reset link "
   "from Users.">>}.
{<<"kb:sso">>, <<"account">>,
 <<"Configure single sign-on">>,
 <<"SSO needs the Team plan. Upload your identity provider "
   "metadata in Settings, Security, then map the email "
   "attribute. Keep one admin with a password as a fallback.">>}.
{<<"kb:two-factor">>, <<"account">>,
 <<"Turn on two-factor authentication">>,
 <<"In Profile, Security, scan the QR code with an authenticator "
   "app and save the recovery codes. Admins can require "
   "two-factor for every user.">>}.
{<<"kb:webhooks">>, <<"integrations">>,
 <<"Receive payment webhooks">>,
 <<"Add an HTTPS endpoint in Settings, Webhooks. Each event is "
   "signed; verify the signature header. Failed deliveries are "
   "retried 5 times over 24 hours.">>}.
{<<"kb:api-keys">>, <<"integrations">>,
 <<"Rotate API keys">>,
 <<"Create the new key first, deploy it, then revoke the old key "
   "in Settings, API. A revoked key stops working at once.">>}.
{<<"kb:plans">>, <<"account">>,
 <<"Change your plan">>,
 <<"Upgrades apply at once and are prorated. Downgrades apply at "
   "the end of the billing period. SSO and audit logs need the "
   "Team plan.">>}.
rebar.config
{erl_opts, [debug_info]}.
%% barrel_rerank is only started when RERANK=1.
{deps, [{barrel, "~> 1.12"},
        {barrel_rerank, "~> 1.0"}]}.
run.sh
#!/bin/sh
# Build, then run main/0 once on a fresh data directory.
#   EMBEDDER=hash (default) | ollama   RERANK=1 RERANK_VENV=<dir>
set -e
cd "$(dirname "$0")"
rebar3 compile >&2
rm -rf data
exec erl -noshell -config config/sys \
  -pa _build/default/lib/*/ebin _build/default/checkouts/*/ebin \
  -eval 'private_rag:main()'

Expected output

The expected.txt of the example, from a real run. Revisions, ids, tokens and timestamps differ on every run; scripts/verify-examples.sh masks them before it compares.

expected.txt
embedder: hash_embed hash-64, 64 dimensions
ingested 10 articles

question: A customer was charged twice this month. How do I
refund one of the payments?

bm25 "refund charged twice":
  5.2942 kb:double-charge
  2.2569 kb:refund

hybrid, billing articles only:
  0.0164 kb:double-charge
  0.0161 kb:refund
  0.0152 kb:export
  0.0152 kb:tax

rerank: skipped (set RERANK=1)

--- prompt (printed, not sent) ---
Answer the question using only the support articles below. Cite
the article id for each step. If the articles do not cover it,
say so.

[kb:double-charge] A customer was charged twice
A double charge usually comes from a retried checkout. Check
Payments on the invoice: if two payments share the same amount
and day, refund the second one. Pending duplicates drop off
within 3 days without a refund.

[kb:refund] Refund a payment
Open the invoice, choose Payments, then Refund next to the
payment. A full or partial refund goes back to the card used.
Card refunds take 5 to 10 business days to appear.

[kb:export] Export invoices as CSV
Admins can export invoices from Reports, Export. Pick a date
range and the CSV format. Exports over 10000 rows are sent by
email.

Question: A customer was charged twice this month. How do I
refund one of the payments?

Up and down to moveEnter to open