chore: tighten repo tooling (#2)

Co-authored-by: Neverdecel <neverdecel@proton.me>

Author: Neverdecel

.github/workflows/ci-tests.yml

@@ -5,32 +5,34 @@ on:
     branches: [ main, master, develop ]
   pull_request:
     branches: [ main, master, develop ]
+  schedule:
+    - cron: "0 3 * * *"
 
 jobs:
   test-imports:
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
-    
+
     - name: Set up Python 3.11
       uses: actions/setup-python@v5
       with:
         python-version: '3.11'
-        
+
     - name: Cache pip dependencies
       uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-
-    
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-    
+
     - name: Test Import Structure
       run: |
         python -c "import coderag.config; print('✓ Config import successful')"
@@ -53,13 +55,14 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          pip install black flake8 isort mypy pytest
-      - name: Lint and type-check
+      - name: Format check
         run: |
           black --check .
           isort --check-only .
-          flake8 . --max-line-length=88 --ignore=E203,W503
-          mypy .
+      - name: Lint
+        run: flake8 . --max-line-length=88 --ignore=E203,W503
+      - name: Type check
+        run: mypy .
       - name: Run tests
         env:
           PYTHONPATH: ${{ github.workspace }}

.pre-commit-config.yaml

@@ -1,36 +1,29 @@
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-added-large-files
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.8.0
     hooks:
       - id: black
         language_version: python3
-        args: ['--line-length=88']
-  
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
-    hooks:
-      - id: flake8
-        args: ['--max-line-length=88', '--ignore=E203,W503']
-  
-  - repo: https://github.com/pycqa/isort
+  - repo: https://github.com/PyCQA/isort
     rev: 5.13.2
     hooks:
       - id: isort
         args: ["--profile", "black"]
-  
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: ["flake8-bugbear==24.4.26"]
+        args: ["--max-line-length=88", "--ignore=E203,W503"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.11.1
     hooks:
       - id: mypy
-        additional_dependencies: [types-all]
-        args: [--ignore-missing-imports, --no-strict-optional]
-  
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
-      - id: check-added-large-files
-      - id: check-merge-conflict
-      - id: debug-statements
+        additional_dependencies: ["types-requests"]
+        args: ["--config-file=pyproject.toml"]

DEVELOPMENT.md

@@ -10,18 +10,22 @@ cd CodeRAG
 python -m venv venv
 source venv/bin/activate  # Windows: venv\Scripts\activate
 pip install -r requirements.txt
+
+> The requirements file delegates to `-e .[dev]`, so you can also run
+> `pip install -e .[dev]` directly if you prefer editable installs.
 ```
 
 ### 2. Configure Pre-commit Hooks
 
 ```bash
 pip install pre-commit
 pre-commit install
+pre-commit run --all-files
 ```
 
 This will run code quality checks on every commit:
 - **Black**: Code formatting
-- **isort**: Import sorting  
+- **isort**: Import sorting
 - **Flake8**: Linting and style checks
 - **MyPy**: Type checking
 - **Basic hooks**: Trailing whitespace, file endings, etc.
@@ -71,11 +75,11 @@ Use concise docstrings for public functions:
 ```python
 def search_code(query: str, k: int = 5) -> List[Dict[str, Any]]:
     \"\"\"Search the FAISS index using a text query.
-    
+
     Args:
         query: The search query text
         k: Number of results to return
-        
+
     Returns:
         List of search results with metadata
     \"\"\"
@@ -94,18 +98,16 @@ streamlit run app.py
 
 ### Code Quality Checks
 ```bash
-# Format code
+pre-commit run --all-files
+```
+
+If you need to run a specific tool locally:
+
+```bash
 black .
 isort .
-
-# Check linting
 flake8 .
-
-# Type checking
 mypy .
-
-# Run all pre-commit checks
-pre-commit run --all-files
 ```
 
 ## Adding New Features
@@ -173,6 +175,13 @@ print(f"Shape: {result.shape if result is not None else 'None'}")
 - Check file permissions
 - Ensure consistent embedding dimensions
 
+## Routine Maintenance
+
+- **Regenerate the FAISS index** after large code refactors: `python scripts/initialize_index.py`.
+- **Rotate environment secrets** by updating `.env` or your deployment variables, then restarting services.
+- **Refresh dependencies** with `pip install --upgrade -r requirements.txt` and run `pre-commit run --all-files` plus `pytest -q`.
+- **Keep hooks current** using `pre-commit autoupdate` followed by a commit once checks pass.
+
 ## Project Structure
 
 ```
@@ -191,4 +200,4 @@ CodeRAG/
 ├── app.py               # Streamlit frontend
 ├── prompt_flow.py       # RAG orchestration
 └── requirements.txt     # Dependencies
-```
+```

README.md

@@ -2,7 +2,7 @@
 
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![Code Quality](https://github.com/neverdecel/CodeRAG/workflows/Code%20Quality/badge.svg)](https://github.com/neverdecel/CodeRAG/actions)
+[![CI Tests](https://github.com/Neverdecel/CodeRAG/actions/workflows/ci-tests.yml/badge.svg?branch=main)](https://github.com/Neverdecel/CodeRAG/actions/workflows/ci-tests.yml)
 
 > **Note**: This POC was innovative for its time, but modern tools like Cursor and Windsurf now apply this principle directly in IDEs. This remains an excellent educational project for understanding RAG implementation.
 
@@ -14,7 +14,7 @@ CodeRAG combines **Retrieval-Augmented Generation (RAG)** with AI to provide int
 
 Most coding assistants work with limited scope, but CodeRAG provides the full context of your project by:
 - **Real-time indexing** of your entire codebase using FAISS vector search
-- **Semantic code search** powered by OpenAI embeddings  
+- **Semantic code search** powered by OpenAI embeddings
 - **Contextual AI responses** that understand your project structure
 
 ## 🚀 Quick Start
@@ -34,14 +34,17 @@ cd CodeRAG
 python -m venv venv
 source venv/bin/activate  # On Windows: venv\\Scripts\\activate
 
-# Install dependencies
+# Install dependencies (installs the package with dev extras)
 pip install -r requirements.txt
 
 # Configure environment
 cp example.env .env
 # Edit .env with your OpenAI API key and settings
 ```
 
+> The requirements file simply references `-e .[dev]`; feel free to run
+> `pip install -e .[dev]` directly if you prefer editable installs.
+
 ### Configuration
 
 Create a `.env` file with your settings:
@@ -63,6 +66,9 @@ python main.py
 
 # In a separate terminal, start the web interface
 streamlit run app.py
+
+# Query the local index from the terminal (after indexing completes)
+coderag-cli "how is faiss configured?"
 ```
 
 ## 📖 How It Works
@@ -88,7 +94,7 @@ graph LR
 
 ```
 CodeRAG/
-├── 🧠 coderag/           # Core RAG functionality  
+├── 🧠 coderag/           # Core RAG functionality
 │   ├── config.py         # Environment configuration
 │   ├── embeddings.py     # OpenAI embedding generation
 │   ├── index.py          # FAISS vector operations
@@ -103,7 +109,7 @@ CodeRAG/
 ### Key Components
 
 - **🔍 Vector Search**: FAISS-powered similarity search for code retrieval
-- **🎯 Smart Embeddings**: OpenAI embeddings capture semantic code meaning  
+- **🎯 Smart Embeddings**: OpenAI embeddings capture semantic code meaning
 - **📡 Real-time Updates**: Watchdog monitors file changes for live indexing
 - **💬 Conversational UI**: Streamlit interface with chat-like experience
 
@@ -116,7 +122,7 @@ CodeRAG/
 "Show me examples of the embedding generation process"
 ```
 
-### Get Improvements  
+### Get Improvements
 ```
 "How can I optimize the search performance?"
 "What are potential security issues in this code?"
@@ -125,7 +131,7 @@ CodeRAG/
 
 ### Debug Issues
 ```
-"Why might the search return no results?"  
+"Why might the search return no results?"
 "How do I troubleshoot OpenAI connection issues?"
 "What could cause indexing to fail?"
 ```
@@ -138,11 +144,7 @@ CodeRAG/
 # Install pre-commit hooks
 pip install pre-commit
 pre-commit install
-
-# Run formatting and linting
-black .
-flake8 .
-mypy .
+pre-commit run --all-files
 ```
 
 ### Testing
@@ -165,7 +167,7 @@ python scripts/run_monitor.py
 - Verify OpenAI API key is working
 - Ensure your query relates to indexed Python files
 
-**OpenAI API errors**  
+**OpenAI API errors**
 - Verify API key in `.env` file
 - Check API usage limits and billing
 - Ensure model names are correct (gpt-4, text-embedding-ada-002)

coderag/cli.py

@@ -0,0 +1,56 @@
+"""Minimal command-line interface for querying an existing CodeRAG index."""
+
+import argparse
+import logging
+import textwrap
+from typing import List
+
+from coderag.search import search_code
+
+
+def _format_result(result: dict, index: int) -> str:
+    snippet = textwrap.shorten(
+        result.get("content", "").replace("\n", " "), width=200, placeholder="..."
+    )
+    return (
+        f"{index}. {result.get('filename')} ({result.get('filepath')})\n"
+        f"   similarity={result.get('distance', 0.0):.3f}\n"
+        f"   {snippet}"
+    )
+
+
+def main(argv: List[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Query a local CodeRAG FAISS index without the Streamlit UI."
+    )
+    parser.add_argument("query", help="Text to search for in the indexed codebase.")
+    parser.add_argument(
+        "-k",
+        type=int,
+        default=5,
+        help="Maximum number of matches to display (defaults to 5).",
+    )
+    parser.add_argument(
+        "--log-level",
+        default="WARNING",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        help="Logging verbosity for debugging issues.",
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=getattr(logging, args.log_level))
+
+    results = search_code(args.query, k=args.k)
+    if not results:
+        print("No results found; ensure the FAISS index exists and contains data.")
+        return 1
+
+    for idx, item in enumerate(results, start=1):
+        print(_format_result(item, idx))
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

coderag/embeddings.py

@@ -3,11 +3,7 @@
 
 import numpy as np
 from openai import OpenAI
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_exponential,
-)
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from coderag.config import OPENAI_API_KEY, OPENAI_EMBEDDING_MODEL
 

example.env

@@ -8,4 +8,4 @@ WATCHED_DIR=/home/user/projects/my_codebase
 
 # FAISS Configuration
 FAISS_INDEX_FILE=/home/user/projects/coderag/faiss_index.bin
-EMBEDDING_DIM=1536
+EMBEDDING_DIM=1536