Merge pull request #7 from nyxCore-Systems/develop

feat: v2.0.0 — ExplainMatch + model provenance

Author: SimplyLiz

CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project are documented here.
 
 ---
 
+## [2.0.0] — 2026-04-13
+
+### Added
+
+**v2.0 — Semantic explainability + model provenance**
+
+- **`ExplainMatch { query, result_uri, top_k, chunk_lines, model }`** — explain *why* `result_uri` ranked as a strong semantic match. Chunks `result_uri`'s source into `chunk_lines`-line windows, embeds each in one batch call, cosine-scores each against the query embedding (cached for URI queries; embedded on the fly for text queries), and returns the top-`top_k` chunks with `(start_line, end_line, chunk_text, score)`. Turns "this file is relevant" into "these specific lines are relevant." Not permitted inside `BatchQuery` (requires HTTP). Returns `ExplainMatchResult { chunks: Vec<ExplanationChunk>, query_model }`.
+- **Model provenance** — every `set_file_embedding` and `set_symbol_embedding` now records the model name that produced the vector. The name is supplied by the `EmbeddingBatch` handler from `embed_texts`'s return value, so it reflects the model actually used (not just what was configured). `QueryFileStatus` now returns `embedding_model: Option<String>`. `QueryIndexStatus` now returns `mixed_models: bool` and `models_in_index: Vec<String>` — clients can warn users when a model upgrade left the index with mixed-model vectors, making cosine scores unreliable across the boundary.
+- **New wire types**: `ExplanationChunk { start_line, end_line, chunk_text, score }`.
+- **1 new MCP tool**: `lip_explain_match`.
+- **MCP updates**: `lip_file_status` response now includes `embedding_model`; `lip_index_status` response now includes `mixed_models` flag and `models_in_index` list with a `⚠ MIXED MODELS` warning in text output.
+
+---
+
 ## [1.9.0] — 2026-04-13
 
 ### Added

Cargo.lock

@@ -7,7 +7,7 @@ members  = [
 ]
 
 [workspace.package]
-version    = "1.9.0"
+version    = "2.0.0"
 edition    = "2021"
 authors    = ["Lisa Welsch <lisa@tastehub.io>"]
 license    = "MIT"

Cargo.toml

@@ -236,6 +236,7 @@ lip query export-embeddings file:///src/auth.rs file:///src/session.rs --output
 | `lip_prune_deleted` | Remove index entries for files no longer on disk (v1.8) |
 | `lip_get_centroid` | Server-side embedding centroid of a file set (v1.9) |
 | `lip_stale_embeddings` | Files whose embedding is older than their current mtime (v1.9) |
+| `lip_explain_match` | Why a result matched: top-scoring chunks of `result_uri` against a query (v2.0) |
 
 **Recommended agent workflow before modifying code:**
 1. `lip_workspace_symbols` — find URIs for all symbols you plan to touch
@@ -377,7 +378,7 @@ Requires Rust 1.78+. No system `protoc` required.
 
 ## Status
 
-v1.9 — Connective tissue: `filter` glob + `min_score` threshold on all nearest-neighbour calls, `GetCentroid` (server-side mean embedding, no raw vectors shipped), `QueryStaleEmbeddings` (mtime vs indexed-at freshness probe). v1.8: `FindBoundaries`, `SemanticDiff`, `QueryNearestInStore` (cross-repo federation), `QueryNoveltyScore`, `ExtractTerminology`, `PruneDeleted`. v1.7: 6 semantic retrieval primitives. v1.6: `ReindexFiles`, `Similarity`, `QueryExpansion`, `Cluster`, `ExportEmbeddings`. Wire format is JSON; FlatBuffers IPC is planned for v2.0 (see roadmap).
+v2.0 — `ExplainMatch` (chunk-level explanation: which lines in a result file drove the match), model provenance (`FileStatus` exposes the embedding model per file; `IndexStatus` warns when the index contains mixed-model vectors). v1.9: `filter` glob + `min_score` on all NN calls, `GetCentroid`, `QueryStaleEmbeddings`. v1.8: `FindBoundaries`, `SemanticDiff`, `QueryNearestInStore` (cross-repo federation), `QueryNoveltyScore`, `ExtractTerminology`, `PruneDeleted`. v1.7: 6 semantic retrieval primitives. v1.6: `ReindexFiles`, `Similarity`, `QueryExpansion`, `Cluster`, `ExportEmbeddings`. Wire format is JSON.
 
 ---
 

README.md

@@ -506,9 +506,9 @@ impl Session {
                     let mut db = self.db.lock().await;
                     for (uri, vec) in miss_uris.iter().zip(new_vecs.iter()) {
                         if uri.starts_with("lip://") {
-                            db.set_symbol_embedding(uri, vec.clone());
+                            db.set_symbol_embedding(uri, vec.clone(), &used_model);
                         } else {
-                            db.set_file_embedding(uri, vec.clone());
+                            db.set_file_embedding(uri, vec.clone(), &used_model);
                         }
                     }
                 }
@@ -564,22 +564,28 @@ impl Session {
                     .as_ref()
                     .as_ref()
                     .map(|c| c.default_model().to_owned());
+                let models_in_index = db.file_embedding_model_names();
+                let mixed_models = models_in_index.len() > 1;
                 ServerMessage::IndexStatusResult {
                     indexed_files,
                     pending_embedding_files: pending,
                     last_updated_ms: last_ms,
                     embedding_model,
+                    mixed_models,
+                    models_in_index,
                 }
             }
 
             ClientMessage::QueryFileStatus { uri } => {
                 let db = self.db.lock().await;
                 let (indexed, has_embedding, age_seconds) = db.file_status(&uri);
+                let embedding_model = db.file_embedding_model(&uri).map(str::to_owned);
                 ServerMessage::FileStatusResult {
                     uri,
                     indexed,
                     has_embedding,
                     age_seconds,
+                    embedding_model,
                 }
             }
 
@@ -706,19 +712,20 @@ impl Session {
                     };
                     // Embed — no db lock held during HTTP call.
                     let texts = vec![embed_text];
-                    let (mut vecs, _) = match client.embed_texts(&texts, model.as_deref()).await {
-                        Ok(r) => r,
-                        Err(e) => {
-                            return ServerMessage::Error {
-                                message: format!("embedding failed: {e}"),
+                    let (mut vecs, sym_model) =
+                        match client.embed_texts(&texts, model.as_deref()).await {
+                            Ok(r) => r,
+                            Err(e) => {
+                                return ServerMessage::Error {
+                                    message: format!("embedding failed: {e}"),
+                                }
                             }
-                        }
-                    };
+                        };
                     let v = vecs.pop().unwrap_or_default();
                     // Cache the computed vector for future calls.
                     {
                         let mut db = self.db.lock().await;
-                        db.set_symbol_embedding(&symbol_uri, v.clone());
+                        db.set_symbol_embedding(&symbol_uri, v.clone(), &sym_model);
                     }
                     v
                 };
@@ -1362,6 +1369,134 @@ impl Session {
                 }
                 ServerMessage::StaleEmbeddingsResult { uris: stale }
             }
+
+            // ── v2.0: ExplainMatch ────────────────────────────────────────
+            ClientMessage::ExplainMatch {
+                query,
+                result_uri,
+                top_k,
+                chunk_lines,
+                model,
+            } => {
+                let Some(client) = self.embedding_client.as_ref().as_ref() else {
+                    return ServerMessage::Error {
+                        message: "embedding not configured — set LIP_EMBEDDING_URL".into(),
+                    };
+                };
+                let effective_top_k = if top_k == 0 { 5 } else { top_k };
+                let chunk_size = if chunk_lines == 0 { 20 } else { chunk_lines };
+
+                // Resolve the query embedding.
+                let (query_vec, query_model) = {
+                    let db = self.db.lock().await;
+                    if let Some(v) = db.get_file_embedding(&query) {
+                        let m = db
+                            .file_embedding_model(&query)
+                            .unwrap_or_else(|| client.default_model())
+                            .to_owned();
+                        (v.clone(), m)
+                    } else {
+                        drop(db);
+                        // Not a cached URI — treat as free-text query.
+                        let texts = vec![query];
+                        match client.embed_texts(&texts, model.as_deref()).await {
+                            Ok((mut vecs, m)) => (vecs.pop().unwrap_or_default(), m),
+                            Err(e) => {
+                                return ServerMessage::Error {
+                                    message: format!("embedding failed: {e}"),
+                                }
+                            }
+                        }
+                    }
+                };
+
+                if query_vec.is_empty() {
+                    return ServerMessage::Error {
+                        message: "could not obtain query embedding".into(),
+                    };
+                }
+
+                // Read source text for result_uri.
+                let source = {
+                    let db = self.db.lock().await;
+                    db.file_source_text(&result_uri).unwrap_or_default()
+                };
+                if source.is_empty() {
+                    return ServerMessage::ExplainMatchResult {
+                        chunks: vec![],
+                        query_model,
+                    };
+                }
+
+                // Chunk the source.
+                let lines: Vec<&str> = source.lines().collect();
+                let raw_chunks: Vec<(u32, u32, String)> = lines
+                    .chunks(chunk_size)
+                    .enumerate()
+                    .map(|(i, chunk_lines_slice)| {
+                        let start = (i * chunk_size) as u32;
+                        let end = (start as usize + chunk_lines_slice.len() - 1) as u32;
+                        (start, end, chunk_lines_slice.join("\n"))
+                    })
+                    .collect();
+
+                if raw_chunks.is_empty() {
+                    return ServerMessage::ExplainMatchResult {
+                        chunks: vec![],
+                        query_model,
+                    };
+                }
+
+                // Embed all chunks in one call.
+                let chunk_texts: Vec<String> =
+                    raw_chunks.iter().map(|(_, _, t)| t.clone()).collect();
+                let (chunk_vecs, chunk_model) =
+                    match client.embed_texts(&chunk_texts, model.as_deref()).await {
+                        Ok(r) => r,
+                        Err(e) => {
+                            return ServerMessage::Error {
+                                message: format!("embedding failed: {e}"),
+                            }
+                        }
+                    };
+                let _ = chunk_model; // we report query_model, not per-chunk model
+
+                // Score each chunk against the query vector.
+                let q_norm: f32 = query_vec.iter().map(|x| x * x).sum::<f32>().sqrt();
+                let mut scored: Vec<crate::query_graph::types::ExplanationChunk> = raw_chunks
+                    .into_iter()
+                    .zip(chunk_vecs.into_iter())
+                    .filter_map(|((start_line, end_line, chunk_text), vec)| {
+                        if vec.len() != query_vec.len() || q_norm == 0.0 {
+                            return None;
+                        }
+                        let v_norm: f32 = vec.iter().map(|x| x * x).sum::<f32>().sqrt();
+                        if v_norm == 0.0 {
+                            return None;
+                        }
+                        let dot: f32 = query_vec.iter().zip(vec.iter()).map(|(a, b)| a * b).sum();
+                        let score = dot / (q_norm * v_norm);
+                        Some(crate::query_graph::types::ExplanationChunk {
+                            start_line,
+                            end_line,
+                            chunk_text,
+                            score,
+                        })
+                    })
+                    .collect();
+
+                scored.sort_by(|a, b| {
+                    b.score
+                        .partial_cmp(&a.score)
+                        .unwrap_or(std::cmp::Ordering::Equal)
+                });
+                scored.truncate(effective_top_k);
+
+                ServerMessage::ExplainMatchResult {
+                    chunks: scored,
+                    query_model,
+                }
+            }
         }
     }
 }
@@ -1523,21 +1658,27 @@ fn process_query_sync(
         // Status queries are read-only and safe inside a batch.
         ClientMessage::QueryIndexStatus => {
             let (indexed_files, pending, last_ms) = db.index_status();
+            let models_in_index = db.file_embedding_model_names();
+            let mixed_models = models_in_index.len() > 1;
             ok(ServerMessage::IndexStatusResult {
                 indexed_files,
                 pending_embedding_files: pending,
                 last_updated_ms: last_ms,
                 embedding_model: None, // no client reference available in sync context
+                mixed_models,
+                models_in_index,
             })
         }
 
         ClientMessage::QueryFileStatus { uri } => {
             let (indexed, has_embedding, age_seconds) = db.file_status(&uri);
+            let embedding_model = db.file_embedding_model(&uri).map(str::to_owned);
             ok(ServerMessage::FileStatusResult {
                 uri,
                 indexed,
                 has_embedding,
                 age_seconds,
+                embedding_model,
             })
         }
 
@@ -1899,6 +2040,10 @@ fn process_query_sync(
         ClientMessage::QueryStaleEmbeddings { .. } => {
             err("QueryStaleEmbeddings requires filesystem I/O; not permitted in BatchQuery")
         }
+
+        ClientMessage::ExplainMatch { .. } => {
+            err("ExplainMatch requires async HTTP; not permitted in BatchQuery")
+        }
     }
 }