Up to date main crawler

backend/bloom/config.py

@@ -2,20 +2,13 @@
 from pathlib import Path
 from datetime import timedelta
 from pydantic_settings import BaseSettings, SettingsConfigDict
-from typing import Any
 
 from pydantic import (
-    AliasChoices,
-    AmqpDsn,
-    BaseModel,
     Field,
-    ImportString,
-    PostgresDsn,
-    RedisDsn,
-    field_validator,
     model_validator
 )
 
+
 class Settings(BaseSettings):
 
     model_config = SettingsConfigDict(
@@ -28,30 +21,30 @@ class Settings(BaseSettings):
         env_ignore_empty=True,
         env_nested_delimiter='__',
         env_file='.env',
-        env_file_encoding = 'utf-8',
+        env_file_encoding='utf-8',
         extra='ignore'
-        )
-    
+    )
+
     # Déclaration des attributs/paramètres disponibles au sein de la class settings
-    postgres_user:str = Field(default='')
-    postgres_password:str = Field(default='')
-    postgres_hostname:str = Field(min_length=1,
-                                  default='localhost')
-    postgres_port:int = Field(gt=1024,
-                                  default=5432)
+    postgres_user: str = Field(default='')
+    postgres_password: str = Field(default='')
+    postgres_hostname: str = Field(min_length=1,
+                                   default='localhost')
+    postgres_port: int = Field(gt=1024,
+                               default=5432)
 
-    postgres_db:str = Field(min_length=1,max_length=32,pattern=r'^(?:[a-zA-Z]|_)[\w\d_]*$')
+    postgres_db: str = Field(min_length=1, max_length=32, pattern=r'^(?:[a-zA-Z]|_)[\w\d_]*$')
     srid: int = Field(default=4326)
-    spire_token:str = Field(default='')
-    data_folder:str=Field(default=str(Path(__file__).parent.parent.parent.joinpath('./data')))
-    db_url:str=Field(default='')
+    spire_token: str = Field(default='')
+    data_folder: str = Field(default=str(Path(__file__).parent.parent.parent.joinpath('./data')))
+    db_url: str = Field(default='')
 
     redis_host: str = Field(default='localhost')
     redis_port: int = Field(default=6379)
     redis_password: str = Field(default='bloom',min_length=1)
     redis_cache_expiration: int = Field(default=900)
 
-    api_pooling_period: timedelta = Field(default=timedelta(minutes=15))
+    api_pooling_period: timedelta = Field(default=timedelta(minutes=2))
 
     logging_level:str=Field(
                                 default="INFO",
@@ -60,15 +53,15 @@ class Settings(BaseSettings):
     api_key:str = Field(min_length=4,default='bloom')
 
     @model_validator(mode='after')
-    def update_db_url(self)->dict:
-        new_url= f"postgresql://{self.postgres_user}:"\
-                          f"{self.postgres_password}@{self.postgres_hostname}:"\
-                          f"{self.postgres_port}/{self.postgres_db}"
+    def update_db_url(self) -> dict:
+        new_url = f"postgresql://{self.postgres_user}:" \
+                  f"{self.postgres_password}@{self.postgres_hostname}:" \
+                  f"{self.postgres_port}/{self.postgres_db}"
         if self.db_url != new_url:
-           self.db_url = new_url
+            self.db_url = new_url
         return self
 
 settings = Settings(_env_file=os.getenv('BLOOM_CONFIG',
-                                    Path(__file__).parent.parent.parent.joinpath('.env')),
+                                        Path(__file__).parent.parent.parent.joinpath('.env')),
                     _secrets_dir=os.getenv('BLOOM_SECRETS_DIR',
-                                    Path(__file__).parent.parent.parent.joinpath('./secrets')))
+                                           Path(__file__).parent.parent.parent.joinpath('./secrets')))

backend/bloom/container.py

@@ -69,14 +69,6 @@ class UseCases(containers.DeclarativeContainer):
         session_factory=db.provided.session,
     )
 
-    get_spire_data_usecase = providers.Factory(GetVesselsFromSpire)
-
-    generate_alert_usecase = providers.Factory(
-        GenerateAlerts,
-        alert_repository=alert_repository,
-        raster_repository=raster_repository,
-    )
-
     spire_ais_data_repository = providers.Factory(
         SpireAisDataRepository,
         session_factory=db.provided.session,
@@ -95,4 +87,13 @@ class UseCases(containers.DeclarativeContainer):
     metrics_repository = providers.Factory(
         MetricsRepository,
         session_factory=db.provided.session,
-    )
+    )
+    get_spire_data_usecase = providers.Factory(
+        GetVesselsFromSpire
+    )
+
+    generate_alert_usecase = providers.Factory(
+        GenerateAlerts,
+        alert_repository=alert_repository,
+        raster_repository=raster_repository,
+    )

backend/bloom/infra/repositories/repository_zone.py

@@ -61,6 +61,14 @@ def get_all_zones_summary(self, session: Session) -> list[ZoneSummary]:
             return []
         return [ZoneRepository.map_to_summary(entity) for entity in q]
 
+    def get_all_zone_categories(self, session: Session) -> list[ZoneCategory]:
+        q = session.query(sql_model.Zone.category,
+                          sql_model.Zone.sub_category).distinct()
+        q=session.execute(q)
+        if not q:
+            return []
+        return [ZoneRepository.map_to_domain(ZoneCategory(category=cat,sub_category=sub)) for cat,sub in q]
+
     def get_all_zone_categories(self, session: Session) -> list[ZoneCategory]:
         q = session.query(sql_model.Zone.category,
                           sql_model.Zone.sub_category).distinct()
@@ -123,6 +131,12 @@ def map_to_summary(zone: sql_model.Zone) -> ZoneSummary:
             created_at=zone.created_at,
             enable=zone.enable,
         )
+    @staticmethod
+    def map_to_domain(category: ZoneCategory) -> ZoneCategory:
+        return ZoneCategory(
+            category=category.category,
+            sub_category=category.sub_category
+        )
 
     @staticmethod
     def map_zonecategory_to_domain(category: ZoneCategory) -> ZoneCategory:

backend/bloom/services/GetVesselsFromSpire.py

@@ -22,6 +22,7 @@ def __init__(self) -> None:
         )
 
     def create_client(self) -> Client:
+        logger.info(f"Connecting to {self.transport.url}...")
         try:
             client = Client(transport=self.transport, fetch_schema_from_transport=True)
         except exceptions.ConnectTimeout:

backend/bloom/tasks/load_dim_zone_amp_from_csv.py

@@ -34,7 +34,7 @@ def map_to_domain(row: pd.Series) -> Zone:
     )
 
 
-def run():
+def run(csv_file_name: str):
     use_cases = UseCases()
     db = use_cases.db()
     zone_repository = use_cases.zone_repository()

backend/bloom/tasks/load_spire_data_from_api.py → backend/load_spire_data_from_api.py

@@ -106,8 +106,8 @@ def run(dump_path: str) -> None:
     )
     args = parser.parse_args()
     time_start = perf_counter()
-    logger.info("DEBUT - Chargement des données JSON depuis l'API SPIRE")
+    logger.info("DEBUT - Chargement des données JSON depuis l'API Kpler")
     run(args.dump_path)
     time_end = perf_counter()
     duration = time_end - time_start
-    logger.info(f"FIN - Chargement des données depuis l'API SPIRE en {duration:.2f}s")
+    logger.info(f"FIN - Chargement des données depuis l'API Kpler en {duration:.2f}s")

docker-compose.yaml

@@ -52,7 +52,7 @@ services:
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD:-redis}
       - REDIS_PORT=${REDIS_PORT:-6379}
-    volumes: 
+    volumes:
       - bloom-redis:/data
     networks:
       - bloom_net
@@ -69,6 +69,8 @@ services:
       - bloom-data:/var/lib/postgresql/data
     networks:
       - bloom_net
+    volumes:
+      - bloom-data:/var/lib/postgresql/data
     healthcheck:
       # PostGis database initialization is done with two steps (postgres+postgis)
       # This causes healthcheck to be valid before real full initialization
@@ -110,7 +112,7 @@ services:
         dockerfile: ./docker/frontend/dev.Dockerfile
         args:
            APP_DIR: /app
-        
+
     volumes:
       #- ./frontend:/app
       #- ./frontend/node_modules:/app/node_modules