Adidas paginated

Author: peter-rauscher

src/solesearch_api/main.py

@@ -80,3 +80,67 @@ async def swagger_ui_html():
 @app.get("/", include_in_schema=False)
 def redirect_to_docs():
     return RedirectResponse(url="/docs")
+
+
+@app.get("/investigate", include_in_schema=False)
+async def investigate_url(url: str, extractor_type: str = "next_json_extractor"):
+    """
+    Endpoint to trigger an investigation download for any URL.
+    This will download the HTML and JSON data and store it in the Investigation folder.
+
+    Args:
+        url: The URL to investigate
+        extractor_type: Type of extractor to use (default: "json")
+    """
+    try:
+        # Import the appropriate extractor
+        # This should be equivalent to:
+        # from solesearch_api.utils.extractors import next_json_extractor as extractor
+        import solesearch_api.utils.extractors as extractor_module
+
+        extractor = getattr(extractor_module, extractor_type)
+
+        # Create an instance of the investigation task
+        from solesearch_api.tasks.scraping.investigation import GenericInvestigationTask
+
+        task_instance = GenericInvestigationTask(
+            url=url,
+            extractor=extractor,
+            headers={
+                "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "accept-language": "en-US,en;q=0.9",
+                "cache-control": "max-age=0",
+                "priority": "u=0, i",
+                "sec-ch-ua": '"Chromium";v="134", "Not:A-Brand";v="24", "Brave";v="134"',
+                "sec-ch-ua-mobile": "?1",
+                "sec-ch-ua-platform": '"Android"',
+                "sec-fetch-dest": "document",
+                "sec-fetch-mode": "navigate",
+                "sec-fetch-site": "same-origin",
+                "sec-fetch-user": "?1",
+                "sec-gpc": "1",
+                "upgrade-insecure-requests": "1",
+                "Referer": "https://www.adidas.com/us/shoes",
+                "Referrer-Policy": "strict-origin-when-cross-origin",
+            },
+        )
+
+        # Force download of HTML and JSON
+        json_data = task_instance.get_json()
+
+        return {
+            "status": "success",
+            "message": f"Investigation data downloaded for URL: {url}",
+            "json_size": len(str(json_data)),
+        }
+    except Exception as e:
+        logfire.error(
+            "Failed to download investigation data",
+            url=url,
+            extractor_type=extractor_type,
+            error=str(e),
+        )
+        return {
+            "status": "error",
+            "message": f"Failed to download investigation data: {str(e)}",
+        }

src/solesearch_api/models/sneaker.py

@@ -12,73 +12,52 @@
 
 
 class SneakerBase(SQLModel):
-    brand: str | None = None
+    id: int | None = Field(default=None, primary_key=True)
     sku: str | None = None
+    brand: str | None = None
     parent_sku: str | None = None
     name: str | None = None
     colorway: str | None = None
     audience: Audience | None = None
     release_date: datetime | None = None
     description: str | None = None
-    retail_price: int | None = None  # Monetary values stored as US cents
+    retail_price: int | None = None
 
 
 class Sneaker(SneakerBase, TimestampedModel, table=True):
     __table_args__ = (
         UniqueConstraint("sku", "brand", name="uix_sneaker_sku_brand"),
         Index("ix_sneaker_sku_brand", "sku", "brand"),
     )
-    id: int | None = Field(default=None, primary_key=True)
     last_observed: datetime | None = None
     stockx_id: str | None = None
     stadium_goods_id: str | None = None
     source: Platform | None = None
     meta: dict = Field(sa_type=JSONB, default_factory=dict)
 
     # Relationships
-    links: list["Link"] = Relationship(back_populates="sneaker", cascade_delete=True)
-    images: list["Image"] = Relationship(back_populates="sneaker", cascade_delete=True)
-    sizes: list["SneakerSize"] = Relationship(
+    sneaker_links: list["Link"] = Relationship(
+        back_populates="sneaker", cascade_delete=True
+    )
+    sneaker_images: list["Image"] = Relationship(
+        back_populates="sneaker", cascade_delete=True
+    )
+    sneaker_sizes: list["SneakerSize"] = Relationship(
         back_populates="sneaker",
         cascade_delete=True,
     )
     nike_launches: list["NikeLaunch"] = Relationship(
         back_populates="sneaker", cascade_delete=True
     )
 
-    def get_links(self) -> list[str]:
-        return [link.url for link in self.links]
-
-    def get_images(self) -> list[str]:
-        return [image.url for image in sorted(self.images, key=lambda i: i.position)]
-
-    def get_sizes(
-        self,
-        size_standard: SizeStandard = SizeStandard.MENS_US,
-    ) -> list[str]:
-        return [size.get_standardized(size_standard) for size in self.sizes]
-
-    @property
-    def prices(self) -> list["Price"]:
-        return list(reduce(lambda x, y: x.prices + y.prices, self.sizes, []))
-
-    def get_prices(self) -> list[int]:
-        return [price.amount for price in self.prices]
-
-    def merge(self, other=None):
-        if other:
-            stockx_images = [
-                img for img in other.images if img.platform == Platform.stockx
-            ]
-            if stockx_images:
-                self.images = stockx_images
-
-            if len(other.colorway) > len(self.colorway):
-                self.colorway = other.colorway
-
 
 class SneakerPublic(SneakerBase):
-    id: int
+    sneaker_links: list["Link"] = Field(default_factory=list, exclude=True)
+    sneaker_images: list["Image"] = Field(default_factory=list, exclude=True)
+
+    sneaker_sizes: list["SneakerSizePublic"] = Field(
+        default_factory=list, alias="sizes"
+    )
 
     @computed_field
     @property
@@ -88,20 +67,17 @@ def retail_price_formatted(self) -> str | None:
         retail_price_in_dollars = Decimal(self.retail_price) / 100
         return f"${retail_price_in_dollars:.2f}"
 
-    @computed_field
-    @property
-    def links(self) -> list[str]:
-        return self.get_links() if hasattr(self, "get_links") else []
-
     @computed_field
     @property
     def images(self) -> list[str]:
-        return self.get_images() if hasattr(self, "get_images") else []
+        return [
+            img.url for img in sorted(self.sneaker_images, key=lambda x: x.position)
+        ]
 
     @computed_field
     @property
-    def sizes(self) -> list[str]:
-        return self.get_sizes() if hasattr(self, "get_sizes") else []
+    def links(self) -> dict[Platform, str]:
+        return {link.platform: link.url for link in self.sneaker_links}
 
 
 class PaginatedSneakersPublic(SQLModel):
@@ -129,9 +105,9 @@ class SneakerSize(TimestampedModel, table=True):
     meta: dict = Field(sa_type=JSONB, default_factory=dict)
 
     sneaker_id: int | None = Field(default=None, foreign_key="sneaker.id")
-    sneaker: Sneaker | None = Relationship(back_populates="sizes")
+    sneaker: Sneaker | None = Relationship(back_populates="sneaker_sizes")
 
-    prices: list["Price"] = Relationship(back_populates="sneaker_size")
+    sneaker_size_prices: list["Price"] = Relationship(back_populates="sneaker_size")
 
 
 class Price(SQLModel, table=True):
@@ -149,7 +125,9 @@ class Price(SQLModel, table=True):
     last_observed: datetime | None = None
 
     sneaker_size_id: int | None = Field(default=None, foreign_key="sneakersize.id")
-    sneaker_size: SneakerSize | None = Relationship(back_populates="prices")
+    sneaker_size: SneakerSize | None = Relationship(
+        back_populates="sneaker_size_prices"
+    )
 
     @property
     def in_dollars(self) -> Decimal:
@@ -167,7 +145,7 @@ class Link(TimestampedModel, table=True):
     url: str
 
     sneaker_id: int | None = Field(default=None, foreign_key="sneaker.id")
-    sneaker: Sneaker | None = Relationship(back_populates="links")
+    sneaker: Sneaker | None = Relationship(back_populates="sneaker_links")
 
 
 class Image(TimestampedModel, table=True):
@@ -181,7 +159,7 @@ class Image(TimestampedModel, table=True):
     url: str
 
     sneaker_id: int | None = Field(default=None, foreign_key="sneaker.id")
-    sneaker: Sneaker | None = Relationship(back_populates="images")
+    sneaker: Sneaker | None = Relationship(back_populates="sneaker_images")
 
 
 class NikeLaunch(SQLModel, table=True):
@@ -205,3 +183,22 @@ class NikeLaunch(SQLModel, table=True):
     merch_product_status: str | None = None
     is_launch_product: bool | None = None
     last_observed: datetime | None = None
+
+
+class SneakerSizePublic(SQLModel):
+    value: str
+    size_standard: SizeStandard
+    sneaker_size_prices: list["PricePublic"] = []
+
+
+class PricePublic(SQLModel):
+    platform: Platform | None = None
+    amount: int
+
+    @computed_field
+    @property
+    def amount_formatted(self) -> str:
+        return f"${Decimal(self.amount) / 100:.2f}"
+
+    last_observed: datetime | None = None
+    first_observed: datetime | None = None

src/solesearch_api/routes/scrape.py

@@ -6,6 +6,8 @@
     adidas_children_and_little_kids_task,
     adidas_youth_and_big_kids_task,
     adidas_new_releases_task,
+    adidas_mens_sneakers_task,
+    adidas_womens_sneakers_task,
     nike_in_stock_scraping_task,
     nike_new_releases_task,
 )
@@ -43,6 +45,8 @@ async def scrape_retailer_new_releases(
         "adidas/baby": adidas_baby_and_toddler_task,
         "adidas/children": adidas_children_and_little_kids_task,
         "adidas/youth": adidas_youth_and_big_kids_task,
+        "adidas/womens": adidas_womens_sneakers_task,
+        "adidas/mens": adidas_mens_sneakers_task,
     }
     if task_name not in task_mapping:
         raise HTTPException(status_code=404, detail="Task not found")

src/solesearch_api/routes/sneakers.py

@@ -79,7 +79,7 @@ async def get_sneakers(
             description="The order to sort in based on the sort key.",
         ),
     ] = SortOrder.DESCENDING,
-) -> Page[Sneaker]:
+) -> Page[SneakerPublic]:
     # set_items_transformer(sneaker_to_public)
     query = select(Sneaker)
 
@@ -125,7 +125,7 @@ async def get_sneakers(
 
 @router.get("/{product_id}", response_model=SneakerPublic)
 async def get_sneaker_by_id(*, db: Session = Depends(get_session), product_id: int):
-    sneaker = db.get(Sneaker, product_id)
+    sneaker = db.exec(select(Sneaker).where(Sneaker.id == product_id)).first()
     if not sneaker:
         raise HTTPException(status_code=404, detail="Sneaker not found")
     return sneaker

src/solesearch_api/tasks/db/base.py

@@ -1,5 +1,6 @@
 import logfire
 from sqlmodel import Session, select
+from sqlalchemy.orm import joinedload
 
 from solesearch_api.models.enums import Platform
 from solesearch_api.models.sneaker import Sneaker
@@ -26,14 +27,22 @@ def create_or_update_sneaker(
             The created or updated sneaker object
         """
         # Check if sneaker already exists with eager loading of relationships
-        statement = select(Sneaker).where(
-            Sneaker.sku == sneaker.sku,
-            Sneaker.brand == sneaker.brand,
+        statement = (
+            select(Sneaker)
+            .where(
+                Sneaker.sku == sneaker.sku,
+                Sneaker.brand == sneaker.brand,
+            )
+            .options(
+                joinedload(Sneaker.sneaker_links),
+                joinedload(Sneaker.sneaker_images),
+                joinedload(Sneaker.sneaker_sizes),
+            )
         )
         existing_sneaker = session.exec(statement).first()
 
         if existing_sneaker:
-            logfire.info(
+            logfire.trace(
                 f"{sneaker.brand} SKU '{sneaker.sku}' found, updating existing sneaker",
                 sneaker=sneaker,
                 existing_sneaker=existing_sneaker,
@@ -65,9 +74,9 @@ def create_or_update_sneaker(
 
             # Add any new links
             existing_links = {
-                link.platform: link.url for link in existing_sneaker.links
+                link.platform: link.url for link in existing_sneaker.sneaker_links
             }
-            for new_link in sneaker.links:
+            for new_link in sneaker.sneaker_links:
                 if existing_links.get(new_link.platform) == new_link.url:
                     # Skip if we already have this exact link
                     continue
@@ -84,14 +93,14 @@ def create_or_update_sneaker(
                 else:
                     # Only add if we don't have a link for this platform yet
                     new_link.sneaker = existing_sneaker
-                    existing_sneaker.links.append(new_link)
+                    existing_sneaker.sneaker_links.append(new_link)
 
             # Add any new images
             existing_images = {
                 (image.platform, image.position): image.url
-                for image in existing_sneaker.images
+                for image in existing_sneaker.sneaker_images
             }
-            for image in sneaker.images:
+            for image in sneaker.sneaker_images:
                 key = (image.platform, image.position)
                 if existing_images.get(key) == image.url:
                     # Skip if we already have this exact image
@@ -109,22 +118,22 @@ def create_or_update_sneaker(
                 else:
                     # Only add if we don't have an image for this platform/position yet
                     image.sneaker = existing_sneaker
-                    existing_sneaker.images.append(image)
+                    existing_sneaker.sneaker_images.append(image)
 
             # Add any new sizes
             existing_sizes = {
                 (size.value, size.size_standard): size
-                for size in existing_sneaker.sizes
+                for size in existing_sneaker.sneaker_sizes
             }
-            for size in sneaker.sizes:
+            for size in sneaker.sneaker_sizes:
                 key = (size.value, size.size_standard)
                 if key in existing_sizes:
                     # Skip if we already have this size
                     continue
                 else:
                     # Only add if we don't have this size yet
                     size.sneaker = existing_sneaker
-                    existing_sneaker.sizes.append(size)
+                    existing_sneaker.sneaker_sizes.append(size)
 
             session.add(existing_sneaker)
             result = existing_sneaker

src/solesearch_api/tasks/scraping/__init__.py

@@ -9,6 +9,12 @@
     adidas_children_and_little_kids_task,
     adidas_youth_and_big_kids_task,
 )
+from solesearch_api.tasks.scraping.retail.adidas.mens_sneakers import (
+    adidas_mens_sneakers_task,
+)
+from solesearch_api.tasks.scraping.retail.adidas.womens_sneakers import (
+    adidas_womens_sneakers_task,
+)
 
 # Nike Tasks
 from solesearch_api.tasks.scraping.retail.nike.new_releases import (