docling docs

docs/AUTHENTICATION.md

@@ -62,18 +62,16 @@ as a reverse proxy to handle OIDC authentication. This approach:
 
 ### Architecture
 
-```text
-┌──────────┐     ┌───────────────┐     ┌─────────────────────┐
-│  User    │────▶│ OAuth2 Proxy  │────▶│ Soliplex Ingester   │
-│ Browser  │     │   (OIDC)      │     │   (API + UI)        │
-└──────────┘     └───────────────┘     └─────────────────────┘
-                        │
-                        ▼
-                 ┌──────────────┐
-                 │ OIDC Provider │
-                 │ (Keycloak,   │
-                 │  Auth0, etc) │
-                 └──────────────┘
+```mermaid
+graph LR
+    User([User Browser]) -->|1. Access App| OAuth2[OAuth2 Proxy<br/>OIDC]
+    OAuth2 -->|2. Authenticated<br/>+ Headers| Ingester[Soliplex Ingester<br/>API + UI]
+    OAuth2 -->|3. Verify Token| OIDC[OIDC Provider<br/>Keycloak, Auth0,<br/>Okta, etc.]
+
+    style User fill:#4a90e2,stroke:#2e5c8a,color:#fff
+    style OAuth2 fill:#3ba13b,stroke:#2d7a2d,color:#fff
+    style Ingester fill:#f39c12,stroke:#c27d0e,color:#fff
+    style OIDC fill:#9b59b6,stroke:#7a4691,color:#fff
 ```
 
 ---

pyproject.toml

@@ -25,7 +25,6 @@ dependencies = [
     "aiosqlite<0.23.0",
     "alembic>=1.17.2",
     "async-lru>=2.0.5",
-    "docling-core>=2.51.1",
     "fastapi[standard]>=0.125.0",
     "fsspec>=2025.12.0",
     "haiku-rag-slim>=0.30.2",

src/soliplex/ingester/cli.py

@@ -316,6 +316,63 @@ def list_batches():
     asyncio.run(_list_batches())
 
 
+async def _check_db(db_name: str, lancedb_dir: str | None):
+    from .lib.models import Database
+    from .lib.operations import check_rag_db_consistency
+
+    await Database.initialize()
+    result = await check_rag_db_consistency(db_name, lancedb_dir)
+
+    print("\n[bold]Database Consistency Check[/bold]")
+    print(f"  db_name:     {result['db_name']}")
+    print(f"  lancedb_dir: {result['lancedb_dir']}")
+    print(f"  db_path:     {result['db_path']}")
+
+    if "error" in result:
+        print(f"\n[red]Error:[/red] {result['error']}")
+
+    print("\n[bold]Summary:[/bold]")
+    print(f"  DocumentDB records: {result['documentdb_count']}")
+    print(f"  LanceDB documents:  {result['lancedb_count']}")
+    print(f"  Matched:            {result['matched']}")
+
+    if result["in_documentdb_only"]:
+        print(f"\n[yellow]In DocumentDB but NOT in LanceDB ({len(result['in_documentdb_only'])}):[/yellow]")
+        for doc in result["in_documentdb_only"]:
+            print(f"  - rag_id: {doc['rag_id']}")
+            print(f"    uri:    {doc['uri']}")
+            print(f"    hash:   {doc['doc_hash']}")
+    else:
+        print("\n[green]No documents in DocumentDB missing from LanceDB[/green]")
+
+    if result["in_lancedb_only"]:
+        print(f"\n[yellow]In LanceDB but NOT in DocumentDB ({len(result['in_lancedb_only'])}):[/yellow]")
+        for doc in result["in_lancedb_only"]:
+            print(f"  - rag_id: {doc['rag_id']}")
+            print(f"    uri:    {doc.get('uri', 'N/A')}")
+            print(f"    title:  {doc.get('title', 'N/A')}")
+    else:
+        print("\n[green]No documents in LanceDB missing from DocumentDB[/green]")
+
+    if not result["in_documentdb_only"] and not result["in_lancedb_only"] and "error" not in result:
+        print("\n[green]✓ Database is consistent[/green]")
+
+
+@app.command("check-db")
+def check_db(
+    db_name: str = typer.Argument(..., help="Database name (data_dir) to check"),
+    lancedb_dir: str = typer.Option(None, "--lancedb-dir", "-l", help="LanceDB directory (uses default if not specified)"),
+):
+    """
+    Check consistency between LanceDB database and DocumentDB records.
+
+    Compares documents in the actual LanceDB database with records tracked
+    in DocumentDB to find discrepancies.
+    """
+    validate_settings(dump=False)
+    asyncio.run(_check_db(db_name, lancedb_dir))
+
+
 @app.command(
     "serve",
 )

src/soliplex/ingester/lib/models.py

@@ -141,6 +141,21 @@ def doc_hash(data: bytes) -> str:
     return f"sha256-{hex_digest}"
 
 
+class DocumentDB(SQLModel, table=True):
+    id: int | None = Field(
+        default=None,
+        primary_key=True,
+        sa_column_kwargs={"autoincrement": True},
+    )
+    doc_hash: str = Field(default=None)  # leave off foreign key due to possible missing documents
+    source: str = Field(default=None)
+    db_name: str = Field(default=None)
+    lancedb_dir: str = Field(default=None)
+    rag_id: str | None = Field(default=None)
+    chunk_count: int | None = Field(default=None)
+    created_date: datetime.datetime = Field(default=None)
+
+
 class DocumentBatch(SQLModel, table=True):
     """
     A batch of documents to be ingested