Fix all logfire logging

Author: peter-rauscher

src/solesearch_api/main.py

@@ -1,6 +1,5 @@
-__version__ = "2.2.0"
+__version__ = "3.0.0"
 
-import logging
 import os
 
 import logfire
@@ -11,15 +10,11 @@
 from fastapi_pagination import add_pagination
 from starlette.middleware.sessions import SessionMiddleware
 
-from solesearch_api.config import ENVIRONMENT
 from solesearch_api.db import initialize_db
 from solesearch_api.routes import auth, scrape, sneakers
 
 desc = """
 ### The Bloomberg Terminal of Sneakers
-- Find product information, from every brand, fast.
-- Never miss another release date.
-- Never buy bricks. Stay ahead of the game with our comprehensive price insights.
 
 [Twitter](https://twitter.com/SoleSearchAPI) | [Github](https://github.com/SoleSearchAPI)
 """
@@ -35,13 +30,12 @@
         "email": "support@solesearch.io",
     },
     description=desc,
-    responses={404: {"description": "Not found"}},  # Custom 404 page
+    responses={404: {"description": "404: Not found"}},  # Custom 404 page
 )
 
 # Configure Logfire
-logfire.configure(environment=ENVIRONMENT)
+logfire.configure(service_name="api")
 logfire.instrument_fastapi(app)
-logger = logging.getLogger(__name__)
 
 # Enable session handling for StocxkX OAuth flow
 app.add_middleware(
@@ -61,15 +55,15 @@
 async def startup_event():
     # Create the database tables
     initialize_db()
-    logger.info("Database initialized")
+    logfire.info("Database initialized")
     # Include all routers
     app.include_router(sneakers.router)
     app.include_router(auth.router)
     app.include_router(scrape.router)
-    logger.info("Routers imported")
+    logfire.info("Routers imported")
     # Load the pagination module
     add_pagination(app)
-    logger.info("Pagination module loaded")
+    logfire.info("Pagination module loaded")
 
 
 @app.get("/docs", include_in_schema=False)

src/solesearch_api/routes/auth.py

@@ -1,8 +1,8 @@
-import logging
 import os
 from datetime import UTC, datetime, timedelta
 from urllib.parse import urlparse
 
+import logfire
 import requests
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import RedirectResponse
@@ -80,8 +80,8 @@ async def stockx_oauth_callback(state: str, code: str, request: Request):
                         seconds=int(tokens["expires_in"]) - 30,
                     )
                 await token.save()
-                logging.info(f"Updated {token_type}")
+                logfire.info(f"Updated {token_type}")
         return tokens
     except Exception as e:
-        logging.exception(e)
+        logfire.exception(e)
         raise HTTPException(status_code=500, detail=str(e))

src/solesearch_api/tasks/__init__.py

@@ -1,6 +1,6 @@
 import logfire
 from celery import Celery
-from celery.signals import beat_init, worker_init
+from celery.signals import worker_init
 
 from solesearch_api.config import CELERY_BACKEND, CELERY_BROKER
 
@@ -11,13 +11,6 @@ class CeleryConfig:
 
 @worker_init.connect()
 def configure_logfire(*args, **kwargs):
-    logfire.configure(service_name="worker")
-    logfire.instrument_celery()
-
-
-@beat_init.connect()
-def init_beat(*args, **kwargs):
-    logfire.configure(service_name="beat")
     logfire.instrument_celery()
 
 

src/solesearch_api/tasks/alerting/healthcheck.py

@@ -1,5 +1,3 @@
-import logging
-
 import requests
 from requests_aws4auth import AWS4Auth
 
@@ -12,8 +10,6 @@
 )
 from solesearch_api.tasks import app
 
-logger = logging.getLogger(__name__)
-
 if ENVIRONMENT == "production":
     auth = AWS4Auth(
         AWS_ACCESS_KEY_ID,

src/solesearch_api/tasks/db/base.py

@@ -1,76 +1,92 @@
+import logfire
 from sqlmodel import Session, select
 
 from solesearch_api.models.sneaker import Sneaker
 from solesearch_api.tasks import app
+from solesearch_api.db import engine
+
+logfire.configure(service_name="worker")
 
 
 @app.task(name="create_or_update_sneaker")
 def create_or_update_sneaker(
-    session: Session,
     sneaker: Sneaker,
 ):
-    """
-    Create a new sneaker or update an existing one with the same SKU and brand.
-
-    This function centralizes the logic for handling sneaker uniqueness based on SKU and brand.
-    It will update all fields of an existing sneaker with the new data, and handle relationships
-    like links, images, sizes, and prices.
+    with Session(engine) as session:
+        """
+        Create a new sneaker or update an existing one with the same SKU and brand.
 
-    Args:
-        session: SQLModel session
-        sneaker: The sneaker object to create or update
+        This function centralizes the logic for handling sneaker uniqueness based on SKU and brand.
+        It will update all fields of an existing sneaker with the new data, and handle relationships
+        like links, images, sizes, and prices.
 
-    Returns:
-        The created or updated sneaker object
-    """
-    # Check if sneaker already exists
-    statement = select(Sneaker).where(
-        Sneaker.sku == sneaker.sku,
-        Sneaker.brand == sneaker.brand,
-    )
-    existing_sneaker = session.exec(statement).first()
+        Args:
+            - sneaker: The sneaker object to create or update
 
-    if existing_sneaker:
-        # Update existing sneaker with new information
-        existing_sneaker.name = sneaker.name or existing_sneaker.name
-        existing_sneaker.parent_sku = sneaker.parent_sku or existing_sneaker.parent_sku
-        existing_sneaker.audience = sneaker.audience or existing_sneaker.audience
-        existing_sneaker.release_date = (
-            sneaker.release_date or existing_sneaker.release_date
-        )
-        existing_sneaker.retail_price = (
-            sneaker.retail_price or existing_sneaker.retail_price
+        Returns:
+            - The created or updated sneaker object
+        """
+        # Check if sneaker already exists with eager loading of relationships
+        statement = select(Sneaker).where(
+            Sneaker.sku == sneaker.sku,
+            Sneaker.brand == sneaker.brand,
         )
-        existing_sneaker.colorway = sneaker.colorway or existing_sneaker.colorway
-        existing_sneaker.description = (
-            sneaker.description or existing_sneaker.description
-        )
-        existing_sneaker.source = existing_sneaker.source or sneaker.source
-        existing_sneaker.stockx_id = existing_sneaker.stockx_id or sneaker.stockx_id
-        existing_sneaker.stadium_goods_id = (
-            existing_sneaker.stadium_goods_id or sneaker.stadium_goods_id
-        )
-
-        # Add any new links
-        existing_link_urls = [link.url for link in existing_sneaker.links]
-        for link in sneaker.links:
-            if link.url not in existing_link_urls:
-                existing_sneaker.links.append(link)
+        existing_sneaker = session.exec(statement).first()
+        if existing_sneaker is None:
+            logfire.info(
+                "Sneaker not found, creating new sneaker",
+                sneaker=sneaker,
+            )
+            # Just add the new sneaker if it doesn't exist already
+            result = sneaker
+        else:
+            logfire.info(
+                "Sneaker found, updating existing sneaker",
+                sneaker=sneaker,
+                existing_sneaker=existing_sneaker,
+            )
+            # Update existing sneaker with new information
+            existing_sneaker.name = sneaker.name or existing_sneaker.name
+            existing_sneaker.parent_sku = (
+                sneaker.parent_sku or existing_sneaker.parent_sku
+            )
+            existing_sneaker.audience = sneaker.audience or existing_sneaker.audience
+            existing_sneaker.release_date = (
+                sneaker.release_date or existing_sneaker.release_date
+            )
+            existing_sneaker.retail_price = (
+                sneaker.retail_price or existing_sneaker.retail_price
+            )
+            existing_sneaker.colorway = sneaker.colorway or existing_sneaker.colorway
+            existing_sneaker.description = (
+                sneaker.description or existing_sneaker.description
+            )
+            existing_sneaker.source = existing_sneaker.source or sneaker.source
+            existing_sneaker.stockx_id = existing_sneaker.stockx_id or sneaker.stockx_id
+            existing_sneaker.stadium_goods_id = (
+                existing_sneaker.stadium_goods_id or sneaker.stadium_goods_id
+            )
 
-        # Add any new images
-        existing_image_urls = [image.url for image in existing_sneaker.images]
-        for image in sneaker.images:
-            if image.url not in existing_image_urls:
-                existing_sneaker.images.append(image)
+            # Add any new links
+            existing_link_urls = {link.url for link in existing_sneaker.links}
+            for link in sneaker.links:
+                if link.url not in existing_link_urls:
+                    existing_sneaker.links.append(link)
 
-        # Add any new sizes
-        existing_size_values = [size.value for size in existing_sneaker.sizes]
-        for size in sneaker.sizes:
-            if size.value not in existing_size_values:
-                existing_sneaker.sizes.append(size)
+            # Add any new images
+            existing_image_urls = {image.url for image in existing_sneaker.images}
+            for image in sneaker.images:
+                if image.url not in existing_image_urls:
+                    session.add(image)
+                    existing_sneaker.images.append(image)
 
-        return existing_sneaker
-    else:
-        # Just add the new sneaker if it doesn't exist already
-        session.add(sneaker)
-        return sneaker
+            # Add any new sizes
+            existing_size_values = {size.value for size in existing_sneaker.sizes}
+            for size in sneaker.sizes:
+                if size.value not in existing_size_values:
+                    session.add(size)
+                    existing_sneaker.sizes.append(size)
+            result = existing_sneaker
+        session.add(result)
+        session.commit()
+        return result

src/solesearch_api/tasks/scraping/base.py

@@ -1,11 +1,11 @@
 import json
-import logging
 import os
 import random
 import time
 from datetime import datetime
 from typing import Callable
 
+import logfire
 import requests
 from requests.adapters import HTTPAdapter
 from sqlmodel import Session
@@ -16,7 +16,7 @@
 from solesearch_api.models.enums import Audience
 from solesearch_api.utils.browser import get_default_browser_headers
 
-logger = logging.getLogger(__name__)
+logfire.configure(service_name="worker")
 
 
 class MisconfigurationError(ValueError):
@@ -49,12 +49,23 @@ def __init__(
         self.extractor = extractor
 
     def run(self, *args, **kwargs):
-        with Session(engine) as session:
-            return self.scrape(session, *args, **kwargs)
+        logfire.info(
+            "Running scraping task",
+            brand=self.brand,
+            task_name=self.__class__.__name__,
+        )
+        result = self.scrape(*args, **kwargs)
+        logfire.info(
+            "Scraping task completed",
+            brand=self.brand,
+            task_name=self.__class__.__name__,
+            result=result,
+        )
+        return result
 
     def guess_audience(self, gender: str | list[str]) -> Audience:
         if gender not in self.audience_mapping:
-            logger.warning(
+            logfire.warning(
                 f"'{gender}' not in {self.__class__.__name__} audience_mapping",
             )
         return self.audience_mapping.get(gender, Audience.UNKNOWN)
@@ -78,8 +89,13 @@ def get_html(
         )
         if os.path.exists(file_path):
             with open(file_path) as f:
+                logfire.info(
+                    "HTML data loaded from file",
+                    brand=self.brand,
+                    task_name=self.__class__.__name__,
+                    file_path=file_path,
+                )
                 return f.read()
-
         # Create a session with retry strategy
         session = requests.Session()
         retry_strategy = Retry(
@@ -99,8 +115,13 @@ def get_html(
         response = session.get(self.download_url, headers=self.headers, timeout=30)
         response.raise_for_status()
         html = response.text
-
         self.save_to_file(html, file_path)
+        logfire.info(
+            "HTML data saved to file",
+            brand=self.brand,
+            task_name=self.__class__.__name__,
+            file_path=file_path,
+        )
         return html
 
     def get_json(self) -> dict:
@@ -112,8 +133,20 @@ def get_json(self) -> dict:
         )
         if os.path.exists(file_path):
             with open(file_path) as f:
+                logfire.info(
+                    "JSON data loaded from file",
+                    brand=self.brand,
+                    task_name=self.__class__.__name__,
+                    file_path=file_path,
+                )
                 return json.load(f)
         html_data = self.get_html()
         json_data = self.extractor(html_data)
         self.save_to_file(json.dumps(json_data, indent=2), file_path)
+        logfire.info(
+            "JSON data saved to file",
+            brand=self.brand,
+            task_name=self.__class__.__name__,
+            file_path=file_path,
+        )
         return json_data

src/solesearch_api/tasks/scraping/retail/adidas/base.py

@@ -1,13 +1,6 @@
-from datetime import datetime, timezone
-import logging
-import re
-from solesearch_api.models.enums import Audience, Platform
-from solesearch_api.models.sneaker import Image, Link, Sneaker
-from solesearch_api.tasks.db.base import create_or_update_sneaker
+from solesearch_api.models.enums import Audience
 from solesearch_api.tasks.scraping.base import BaseScrapingTask
 
-logger = logging.getLogger(__name__)
-
 
 class AdidasScrapingTask(BaseScrapingTask):
     def __init__(self, *args, **kwargs):