feat: persistent findings storage (#22)

Author: GangGreenTemperTatum

README.md

@@ -23,6 +23,7 @@ Experimenting with yarrr' Burp Proxy tab going brrrrrrrrrrrrr.
 
 - [burpference](#burpference)
   - [Prerequisites](#prerequisites)
+  - [Project Structure](#project-structure)
   - [Setup Guide](#setup-guide)
     - [1. Download and Install Burp Suite](#1-download-and-install-burp-suite)
     - [2. Download and import Jython standalone JAR file](#2-download-and-import-jython-standalone-jar-file)
@@ -69,7 +70,25 @@ Some key features:
   - Injection vulnerability assessment across all vectors
   - Additional templates can be created for specific testing scenarios
     - Dynamic prompt switching during runtime to tailor analysis based on target endpoints
+- **Persistent Findings Storage**: All security findings are automatically stored and tracked:
+  - Findings are saved to `logs/findings.json` in a structured format (example below)
+  - Each finding includes timestamp, severity, details, and affected URLs and persist across Burp Suite sessions
+  - Findings are synchronized between Scanner and Proxy analysis
+
+  ```json
+  [
+    {
+      "detail": "{u'<FINDING>'model': u'<provider>/<model>'}",
+      "host": "www.evil.io",
+      "name": "burpference: <SEVERITY> Security Finding",
+      "severity": "<SEVERITY>",
+      "timestamp": "2025-02-09T19:35:56.667000",
+      "url": "https://www.evil.io:443/"
+    }
+  ]
+  ```
 
+<br>
 So grab yer compass, hoist the mainsail, and let **burpference** be yer guide as ye plunder the seven seas of HTTP traffic! Yarrr'!
 
 ---
@@ -98,6 +117,32 @@ Before using **Burpference**, ensure you have the following:
 
 ---
 
+## Project Structure
+```
+/Users/ads/git/burpference/
+├── burpference/                 # Main package directory
+│   ├── __init__.py             # Package initialization
+│   ├── api_adapters.py         # API providers and adapters
+│   ├── assets/                 # Internal assets
+│   │   └── squid_ascii.txt     # ASCII art for extension
+│   ├── burpference.py          # Main extension code
+│   ├── consts.py              # Constants and configurations
+│   ├── db_manager.py          # Database operations
+│   ├── issues.py              # Burp issue implementations
+│   └── scanner.py             # Security scanner functionality
+├── configs/                    # API configuration files
+│   └── *.json                 # JSON config files per provider
+├── logs/                      # Log output directory
+│   ├── findings.json          # Security findings database
+│   └── *.txt                  # Generated log files
+├── prompts/                   # Prompt template files
+│   ├── proxy_prompt.txt       # Default proxy analysis prompt
+│   ├── scanner_prompt.txt     # Scanner analysis prompt
+│   └── openapi_prompt.txt     # OpenAPI analysis prompt
+├── LICENSE                    # Project license
+└── README.md                 # Project documentation
+```
+
 ## Setup Guide
 
 ### 1. Download and Install Burp Suite

burpference/__init__.py

@@ -17,6 +17,7 @@
 from consts import *
 from api_adapters import get_api_adapter
 from issues import BurpferenceIssue
+from db_manager import BurpDBManager
 from threading import Thread
 from scanner import BurpferenceScanner
 
@@ -56,6 +57,7 @@ def __init__(self):
         self.log_message("Extension initialized and running.")
         self._hosts = set()
         self.scanner = None  # Will initialize after callbacks
+        self.db_manager = BurpDBManager()
 
     def registerExtenderCallbacks(self, callbacks):
         self._callbacks = callbacks
@@ -838,7 +840,6 @@ def create_scan_issue(self, messageInfo, processed_response):
             severity = self.extract_severity_from_response(processed_response)
             burp_severity = self.map_severity(severity)
 
-            # Convert response to string and handle escaping
             if isinstance(processed_response, str):
                 detail = processed_response
             else:
@@ -847,7 +848,6 @@ def create_scan_issue(self, messageInfo, processed_response):
             if detail.startswith('"') and detail.endswith('"'):
                 detail = detail[1:-1]
 
-            # Create properly formatted issue name
             issue_name = "burpference: %s Security Finding" % severity
 
             issue = BurpferenceIssue(
@@ -860,6 +860,17 @@ def create_scan_issue(self, messageInfo, processed_response):
                 confidence="Certain"
             )
 
+            finding_dict = {
+                "timestamp": datetime.now().isoformat(),
+                "name": issue_name,
+                "severity": burp_severity,
+                "detail": detail,
+                "url": str(issue.getUrl()),
+                "host": messageInfo.getHttpService().getHost()
+            }
+            self.db_manager.add_finding(finding_dict)
+            self.log_message("Saved finding to database")
+
             self._callbacks.addScanIssue(issue)
             self.log_message("Added %s issue to Burp Scanner" % severity)
 

assets/squid_ascii.txt burpference/assets/squid_ascii.txtassets/squid_ascii.txt renamed to burpference/assets/squid_ascii.txt

@@ -1,15 +1,16 @@
 import os
 from java.awt import Color
 
-# Get the directory containing burpference.py
-EXTENSION_DIR = os.path.dirname(os.path.abspath(__file__))
-ROOT_DIR = os.path.dirname(EXTENSION_DIR)  # Parent directory of burpference folder
-# Define paths relative to ROOT_DIR
-CONFIG_DIR = os.path.join(ROOT_DIR, "configs")
-LOG_DIR = os.path.join(ROOT_DIR, "logs")
-PROMPTS_DIR = os.path.join(ROOT_DIR, "prompts")
+BURPFERENCE_DIR = os.path.dirname(
+    os.path.abspath(__file__)
+)  # /path/to/burpference/burpference
+PROJECT_ROOT = os.path.dirname(BURPFERENCE_DIR)  # /path/to/burpference
+
+CONFIG_DIR = os.path.join(PROJECT_ROOT, "configs")  # /path/to/burpference/configs
+LOG_DIR = os.path.join(PROJECT_ROOT, "logs")  # /path/to/burpference/logs
+PROMPTS_DIR = os.path.join(PROJECT_ROOT, "prompts")  # /path/to/burpference/prompts
 PROXY_PROMPT = os.path.join(PROMPTS_DIR, "proxy_prompt.txt")
-SQUID_ASCII_FILE = os.path.join(ROOT_DIR, "assets", "squid_ascii.txt")
+SQUID_ASCII_FILE = os.path.join(BURPFERENCE_DIR, "assets", "squid_ascii.txt")
 
 # Color constants
 DREADNODE_ORANGE = Color(255, 140, 0)

burpference/burpference.py

@@ -0,0 +1,64 @@
+import os
+import json
+from datetime import datetime
+from consts import PROJECT_ROOT, LOG_DIR
+
+
+class BurpDBManager:
+    def __init__(self, db_path=None):
+        if db_path is None:
+            db_path = os.path.join(LOG_DIR, "findings.json")
+        self.db_path = db_path
+        print("[*] DB Path: %s" % self.db_path)
+        self._ensure_db_dir()
+        self._findings = self._load_findings()
+        print("[*] Loaded %d findings" % len(self._findings))
+
+    def _ensure_db_dir(self):
+        """Ensure the database directory exists"""
+        db_dir = os.path.dirname(self.db_path)
+        if not os.path.exists(db_dir):
+            os.makedirs(db_dir)
+
+    def _load_findings(self):
+        """Load findings from JSON file"""
+        if os.path.exists(self.db_path):
+            try:
+                with open(self.db_path, "r") as f:
+                    return json.load(f)
+            except Exception as e:
+                print("[!] Error loading findings: %s" % str(e))
+                return []
+        return []
+
+    def _save_findings(self):
+        """Save findings to JSON file"""
+        try:
+            with open(self.db_path, "w") as f:
+                json.dump(self._findings, f, indent=2)
+            print("[+] Saved %d findings to %s" % (len(self._findings), self.db_path))
+        except Exception as e:
+            print("[!] Error saving findings: %s" % str(e))
+
+    def add_finding(self, finding):
+        """Add a new finding"""
+        finding["timestamp"] = datetime.now().isoformat()
+        self._findings.append(finding)
+        self._save_findings()
+
+    def get_findings(self, filters=None):
+        """Retrieve findings with optional filters"""
+        if not filters:
+            return self._findings
+
+        filtered = []
+        for finding in self._findings:
+            matches = True
+            for key, value in filters.items():
+                if key not in finding or finding[key] != value:
+                    matches = False
+                    break
+            if matches:
+                filtered.append(finding)
+
+        return filtered

burpference/consts.py

@@ -1,4 +1,5 @@
 from burp import IScanIssue
+import json
 
 
 class BurpferenceIssue(IScanIssue):
@@ -45,3 +46,23 @@ def getHttpMessages(self):
 
     def getHttpService(self):
         return self._httpService
+
+    def to_dict(self):
+        """Convert issue to dictionary for database storage"""
+        return {
+            "name": self._name,
+            "detail": self._detail,
+            "severity": self._severity,
+            "confidence": self._confidence,
+            "host": self._httpService.getHost(),
+            "url": str(self._url),
+            "request_response": json.dumps(
+                [
+                    {
+                        "request": msg.getRequest().tostring(),
+                        "response": msg.getResponse().tostring(),
+                    }
+                    for msg in self._httpMessages
+                ]
+            ),
+        }

burpference/db_manager.py

@@ -20,6 +20,8 @@
 import re
 from datetime import datetime
 from javax.swing.border import EmptyBorder
+from db_manager import BurpDBManager
+from issues import BurpferenceIssue
 
 SCANNER_PROMPT = os.path.join(
     os.path.dirname(os.path.dirname(__file__)), "prompts", "scanner_prompt.txt"
@@ -44,6 +46,7 @@ def __init__(self, callbacks, helpers, config, api_adapter, colors=None):
         self.LIGHTER_BACKGROUND = self.colors.get("LIGHTER_BACKGROUND")
         self.DREADNODE_GREY = self.colors.get("DREADNODE_GREY")
         self.DREADNODE_ORANGE = self.colors.get("DREADNODE_ORANGE")
+        self.db_manager = BurpDBManager()
 
     def add_host(self, host):
         """Add a host to the scanner's tracked hosts"""
@@ -503,3 +506,24 @@ def update_config_display(self):
         if hasattr(self, "config_label"):
             self.config_label.setText(self.get_config_status())
             self.refresh_prompt_template()
+
+    def create_scan_issue(self, messageInfo, processed_response):
+        try:
+            issue = BurpferenceIssue(
+                httpService=self._helpers.buildHttpService(
+                    host, port, protocol == "https"
+                ),
+                url=url,
+                httpMessages=[messageInfo],
+                name="burpference Scanner: %s Finding" % level,
+                detail=analysis,
+                severity=severity,
+                confidence="Certain",
+            )
+
+            self.db_manager.add_finding(issue.to_dict())
+
+            self._callbacks.addScanIssue(issue)
+
+        except Exception as e:
+            self._callbacks.printOutput("Error creating scan issue: %s" % str(e))