updates!

Author: peter-rauscher

src/solesearch_api/main.py

@@ -1,5 +1,6 @@
 __version__ = "2.2.0"
 
+import logging
 import os
 
 import logfire
@@ -39,6 +40,7 @@
 # Configure Logfire
 logfire.configure()
 logfire.instrument_fastapi(app)
+logger = logging.getLogger(__name__)
 
 # Enable session handling for StocxkX OAuth flow
 app.add_middleware(
@@ -58,12 +60,15 @@
 async def startup_event():
     # Create the database tables
     initialize_db()
+    logger.info("Database initialized")
     # Include all routers
     app.include_router(sneakers.router)
     app.include_router(auth.router)
     app.include_router(scrape.router)
+    logger.info("Routers imported")
     # Load the pagination module
     add_pagination(app)
+    logger.info("Pagination module loaded")
 
 
 @app.get("/docs", include_in_schema=False)

src/solesearch_api/models/enums.py

@@ -11,16 +11,19 @@ class Platform(str, Enum):
 
 class Audience(str, Enum):
     UNISEX = "Unisex"
-    MEN = "Men"
-    WOMEN = "Women"
-    YOUTH = "Youth"
+    MEN = "Mens"
+    WOMEN = "Womens"
+    GRADE_SCHOOL = "Grade School"
+    PRESCHOOL = "Preschool"
     TODDLER = "Toddler"
+    YOUTH = "Youth"
     UNKNOWN = "Unknown"
 
 
 class SizeStandard(str, Enum):
     MENS_US = "mens_US"
     WOMENS_US = "womens_US"
+    KIDS_US = "kids_US"
 
 
 class SneakerSortKey(str, Enum):

src/solesearch_api/models/sneaker.py

@@ -1,9 +1,11 @@
 from datetime import datetime
 from decimal import Decimal
 from functools import reduce
+from typing import Any
 
 from pydantic import computed_field
 from sqlalchemy import Index, UniqueConstraint
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field, Relationship, SQLModel
 
 from solesearch_api.models.base import TimestampedModel
@@ -28,9 +30,11 @@ class Sneaker(SneakerBase, TimestampedModel, table=True):
         Index("ix_sneaker_sku_brand", "sku", "brand"),
     )
     id: int | None = Field(default=None, primary_key=True)
+    last_observed: datetime | None = None
     stockx_id: str | None = None
     stadium_goods_id: str | None = None
     source: Platform | None = None
+    meta: dict = Field(sa_type=JSONB, default_factory=dict)
 
     # Relationships
     links: list["Link"] = Relationship(back_populates="sneaker", cascade_delete=True)
@@ -39,6 +43,9 @@ class Sneaker(SneakerBase, TimestampedModel, table=True):
         back_populates="sneaker",
         cascade_delete=True,
     )
+    nike_launches: list["NikeLaunch"] = Relationship(
+        back_populates="sneaker", cascade_delete=True
+    )
 
     def get_links(self) -> list[str]:
         return [link.url for link in self.links]
@@ -109,31 +116,38 @@ class PaginatedSneakersPublic(SQLModel):
 
 class SneakerSize(TimestampedModel, table=True):
     __table_args__ = (
-        UniqueConstraint("sneaker_id", "value"),
-        Index("ix_sneaker_id_value", "sneaker_id", "value"),
+        UniqueConstraint("sneaker_id", "value", "size_standard"),
+        Index(
+            "ix_sneaker_id_value_size_standard",
+            "sneaker_id",
+            "value",
+            "size_standard",
+        ),
     )
     id: int | None = Field(default=None, primary_key=True)
-    value: int
+    value: str
+    size_standard: SizeStandard
+    meta: dict = Field(sa_type=JSONB, default_factory=dict)
 
     sneaker_id: int | None = Field(default=None, foreign_key="sneaker.id")
     sneaker: Sneaker | None = Relationship(back_populates="sizes")
 
     prices: list["Price"] = Relationship(back_populates="sneaker_size")
 
-    def get_standardized(self, size_standard: SizeStandard = SizeStandard.MENS_US):
-        if size_standard == SizeStandard.MENS_US:
-            return self.value
-
 
-class Price(TimestampedModel, table=True):
+class Price(SQLModel, table=True):
+    # Ordered by last_observed timestamp
     __table_args__ = (
-        UniqueConstraint("sneaker_size_id", "platform"),
         Index("ix_sneaker_size_id_platform", "sneaker_size_id", "platform"),
+        Index("ix_amount", "amount"),
+        Index("ix_last_observed", "last_observed"),
+        Index("ix_first_observed", "first_observed"),
     )
     id: int | None = Field(default=None, primary_key=True)
     platform: Platform | None = None
     amount: int  # Monetary values stored as US cents
-    observed_at: datetime | None = None
+    first_observed: datetime | None = None
+    last_observed: datetime | None = None
 
     sneaker_size_id: int | None = Field(default=None, foreign_key="sneakersize.id")
     sneaker_size: SneakerSize | None = Relationship(back_populates="prices")
@@ -142,10 +156,6 @@ class Price(TimestampedModel, table=True):
     def in_dollars(self) -> Decimal:
         return Decimal(self.amount) / 100
 
-    @property
-    def for_size(self) -> int:
-        return self.sneaker_size.value
-
 
 class Link(TimestampedModel, table=True):
     __table_args__ = (
@@ -173,3 +183,26 @@ class Image(TimestampedModel, table=True):
 
     sneaker_id: int | None = Field(default=None, foreign_key="sneaker.id")
     sneaker: Sneaker | None = Relationship(back_populates="images")
+
+
+class NikeLaunch(SQLModel, table=True):
+    __table_args__ = (
+        UniqueConstraint("product_id", "launch_id"),
+        Index("ix_product_id_launch_id", "product_id", "launch_id"),
+    )
+    id: int | None = Field(default=None, primary_key=True)
+
+    sneaker_id: int | None = Field(default=None, foreign_key="sneaker.id")
+    sneaker: Sneaker | None = Relationship(back_populates="nike_launches")
+
+    launch_id: str
+    product_id: str
+    launch_method: str | None = None
+    payment_method: str | None = None
+    launch_link: str | None = None
+    start_entry_date: datetime | None = None
+    stop_entry_date: datetime | None = None
+    stop_notify_date: datetime | None = None
+    merch_product_status: str | None = None
+    is_launch_product: bool | None = None
+    last_observed: datetime | None = None

src/solesearch_api/routes/scrape.py

@@ -1,11 +1,9 @@
 from celery.result import AsyncResult
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, HTTPException, Request
 
-from solesearch_api.tasks.scraping.retail.adidas.new_releases import (
-    adidas_new_releases_task,
-)
-from solesearch_api.tasks.scraping.retail.adidas.pdp import adidas_pdp_task
-from solesearch_api.tasks.scraping.retail.nike import nike_scraping_task
+from solesearch_api.tasks.scraping import adidas_new_releases_task
+from solesearch_api.tasks.scraping import nike_scraping_task
+from solesearch_api.tasks.scraping import nike_instock_scraping_task
 
 router = APIRouter(
     prefix="/scrape",
@@ -28,21 +26,18 @@ def task_result_factory(request: Request, task: AsyncResult):
     }
 
 
-@router.get("/nike")
-async def scrape_nike_new_releases(request: Request):
-    task = nike_scraping_task.apply_async()
-    return task_status_factory(request, task)
-
-
-@router.get("/adidas")
-async def scrape_adidas_new_releases(request: Request):
-    task = adidas_new_releases_task.apply_async()
-    return task_status_factory(request, task)
-
-
-@router.get("/adidas/{slug}")
-async def scrape_adidas_pdp(request: Request, slug: str):
-    task = adidas_pdp_task.apply_async(args=[slug])
+@router.get("/{retailer}")
+async def scrape_retailer_new_releases(
+    request: Request, retailer: str, nocache: bool = False
+):
+    task_mapping = {
+        "nike": nike_scraping_task,
+        "nike_instock": nike_instock_scraping_task,
+        "adidas": adidas_new_releases_task,
+    }
+    if retailer not in task_mapping:
+        raise HTTPException(status_code=404, detail="Retailer not found")
+    task = task_mapping[retailer].delay(nocache=nocache)
     return task_status_factory(request, task)
 
 

src/solesearch_api/tasks/scraping/__init__.py

@@ -3,4 +3,7 @@
     adidas_new_releases_task,
 )
 from solesearch_api.tasks.scraping.retail.adidas.pdp import adidas_pdp_task
-from solesearch_api.tasks.scraping.retail.nike import nike_scraping_task
+from solesearch_api.tasks.scraping.retail.nike.new_releases import nike_scraping_task
+from solesearch_api.tasks.scraping.retail.nike.in_stock import (
+    nike_instock_scraping_task,
+)

src/solesearch_api/tasks/scraping/base.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 from typing import Callable
 
-import logfire
+import logging
 import requests
 from requests.adapters import HTTPAdapter
 from sqlmodel import Session
@@ -16,6 +16,8 @@
 from solesearch_api.models.enums import Audience
 from solesearch_api.utils.browser import get_default_browser_headers
 
+logger = logging.getLogger(__name__)
+
 
 class MisconfigurationError(ValueError):
     pass
@@ -52,7 +54,7 @@ def run(self, *args, **kwargs):
 
     def guess_audience(self, gender: str | list[str]) -> Audience:
         if gender not in self.audience_mapping:
-            logfire.warning(
+            logger.warning(
                 f"'{gender}' not in {self.__class__.__name__} audience_mapping",
             )
         return self.audience_mapping.get(gender, Audience.UNKNOWN)

src/solesearch_api/tasks/scraping/retail/adidas/base.py

@@ -9,7 +9,7 @@ def __init__(self, *args, **kwargs):
             audience_mapping={
                 "M": Audience.MEN,
                 "W": Audience.WOMEN,
-                "K": Audience.YOUTH,
+                "K": Audience.GRADE_SCHOOL,
                 "U": Audience.UNISEX,
             },
             headers={

src/solesearch_api/tasks/scraping/retail/adidas/new_releases.py

@@ -1,15 +1,16 @@
+import logging
 import re
 from datetime import datetime, timezone
 
-import logfire
-
 from solesearch_api.models.enums import Platform
 from solesearch_api.models.sneaker import Image, Link, Sneaker
 from solesearch_api.tasks.db.base import create_or_update_sneaker
 from solesearch_api.tasks.scraping.retail.adidas.base import AdidasScrapingTask
 from solesearch_api.tasks.scraping.task_registry import register_scraping_task
 from solesearch_api.utils.extractors import next_json_extractor
 
+logger = logging.getLogger(__name__)
+
 
 class AdidasNewReleasesScrapingTask(AdidasScrapingTask):
     name = "solesearch_api.tasks.scraping.retail.adidas.new_releases"
@@ -35,7 +36,7 @@ def scrape(self, session, *args, **kwargs):
     def json_to_sneaker(self, data: dict) -> Sneaker | None:
         sku = data.get("id", "").strip()
         if not sku:
-            logfire.warning(f"No SKU found for {self.brand} product: {data}")
+            logger.warning(f"No SKU found for {self.brand} product: {data}")
             return None
 
         release_date = data.get("preview_to", "").strip()

src/solesearch_api/tasks/scraping/retail/nike/in_stock.py

@@ -0,0 +1,26 @@
+from solesearch_api.models.enums import Audience
+from solesearch_api.tasks.scraping.base import BaseScrapingTask
+from solesearch_api.tasks.scraping.task_registry import register_scraping_task
+from solesearch_api.utils.extractors import next_json_extractor
+
+
+class NikeInStockScrapingTask(BaseScrapingTask):
+    name = "solesearch_api.tasks.scraping.NikeInStockScrapingTask"
+
+    def __init__(self):
+        super().__init__(
+            brand="NikeInStock",
+            download_url="https://www.nike.com/launch?s=in-stock",
+            audience_mapping={
+                "M": Audience.MEN,
+                "W": Audience.WOMEN,
+            },
+            extractor=next_json_extractor,
+        )
+
+    def scrape(self, session, *args, **kwargs):
+        json_data = self.get_json()
+        return json_data
+
+
+nike_instock_scraping_task = register_scraping_task(NikeInStockScrapingTask)