ESS-3444 Moving to Open Telemetry from Open Census (#139)

* Moving to Open Telemetry from Open Census
* Only enable logger when environment variable is set

Author: bjorn-einar-bjartnes-4ss

datareservoirio/_logging.py

@@ -1,14 +1,15 @@
 import logging
 import os
-from functools import wraps, cache
+from functools import cache, wraps
 
-from opencensus.ext.azure.log_exporter import AzureLogHandler
+from azure.monitor.opentelemetry import configure_azure_monitor
 
 import datareservoirio as drio
 
 from ._constants import ENV_VAR_ENABLE_APP_INSIGHTS, ENV_VAR_ENGINE_ROOM_APP_ID
 from .globalsettings import environment
 
+
 @cache
 def get_exceptions_logger() -> logging.Logger:
     exceptions_logger = logging.getLogger(__name__ + "_exception_logger")
@@ -17,13 +18,15 @@ def get_exceptions_logger() -> logging.Logger:
     if os.getenv(ENV_VAR_ENABLE_APP_INSIGHTS) is not None:
         enable_app_insights = os.environ[ENV_VAR_ENABLE_APP_INSIGHTS].lower()
         if enable_app_insights == "true" or enable_app_insights == "1":
-            app_insight_handler = AzureLogHandler(
-                connection_string=environment._application_insight_connectionstring
+            configure_azure_monitor(
+                connection_string=environment._application_insight_connectionstring,
+                logger_name=__name__ + "_exceptions_logger",
             )
-            app_insight_handler.setLevel("WARNING")
-            exceptions_logger.addHandler(app_insight_handler)
+            exceptions_logger.setLevel("WARNING")
+
     return exceptions_logger
 
+
 def log_decorator(log_level):
     def decorator(func):
         @wraps(func)

datareservoirio/client.py

@@ -1,18 +1,19 @@
 import logging
+import os
 import time
 import warnings
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
-from functools import wraps, cache
+from functools import cache, wraps
 from operator import itemgetter
 from urllib.parse import urlencode
 from uuid import uuid4
 
 import numpy as np
 import pandas as pd
 import requests
-from opencensus.ext.azure.log_exporter import AzureLogHandler
+from azure.monitor.opentelemetry import configure_azure_monitor
 from tenacity import (
     retry,
     retry_if_exception_type,
@@ -22,22 +23,30 @@
 )
 from tqdm.auto import tqdm
 
+from datareservoirio._constants import ENV_VAR_ENABLE_APP_INSIGHTS
+
 from ._logging import log_decorator
 from ._utils import function_translation, period_translation
 from .globalsettings import environment
 from .storage import Storage
 
 log = logging.getLogger(__name__)
 
+
 @cache
 def metric() -> logging.Logger:
     logger = logging.getLogger(__name__ + "_metric_appinsight")
-    logger.setLevel(logging.DEBUG)
-    logger.addHandler(
-        AzureLogHandler(connection_string=environment._application_insight_connectionstring)
-    )
+    if os.getenv(ENV_VAR_ENABLE_APP_INSIGHTS) is not None:
+        enable_app_insights = os.environ[ENV_VAR_ENABLE_APP_INSIGHTS].lower()
+        if enable_app_insights == "true" or enable_app_insights == "1":
+            logger.setLevel(logging.DEBUG)
+            configure_azure_monitor(
+                connection_string=environment._application_insight_connectionstring,
+                logger_name=__name__ + "_metric_appinsight",
+            )
     return logger
 
+
 # Default values to push as start/end dates. (Limited by numpy.datetime64)
 _END_DEFAULT = 9214646400000000000  # 2262-01-01
 _START_DEFAULT = -9214560000000000000  # 1678-01-01
@@ -324,13 +333,11 @@ def wrapper(self, series_id, start=None, end=None, **kwargs):
                 ).isoformat()
             number_of_samples = len(result)
             properties = {
-                "custom_dimensions": {
-                    "series_id": series_id,
-                    "start": start_date_as_str,
-                    "end": end_date_as_str,
-                    "elapsed": elapsed_time,
-                    "number-of-samples": number_of_samples,
-                }
+                "series_id": series_id,
+                "start": start_date_as_str,
+                "end": end_date_as_str,
+                "elapsed": elapsed_time,
+                "number-of-samples": number_of_samples,
             }
             metric().info("Timer", extra=properties)
             return result

pyproject.toml

@@ -4,34 +4,32 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datareservoirio"
-authors = [
-    { name="4Subsea", email="support@4subsea.com" }
-]
+authors = [{ name = "4Subsea", email = "support@4subsea.com" }]
 dynamic = ["version"]
 description = "DataReservoir.io Python API"
 readme = "README.rst"
-license = { file="LICENSE" }
+license = { file = "LICENSE" }
 requires-python = ">3.10"
 classifiers = [
-    "Development Status :: 5 - Production/Stable",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
+  "Development Status :: 5 - Production/Stable",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "numpy",
-    "oauthlib",
-    "pandas",
-    "pyarrow",
-    "requests",
-    "requests-oauthlib",
-    "importlib_resources",
-    "opencensus-ext-azure",
-    "tenacity<8.5",
-    "urllib3 > 2",
-    "tqdm"
+  "numpy",
+  "oauthlib",
+  "pandas",
+  "pyarrow",
+  "requests",
+  "requests-oauthlib",
+  "importlib_resources",
+  "tenacity<8.5",
+  "urllib3 > 2",
+  "tqdm",
+  "azure-monitor-opentelemetry",
 ]
 
 [project.urls]
@@ -45,7 +43,7 @@ include = ["datareservoirio*"]
 namespaces = false
 
 [tool.setuptools.dynamic]
-version = {attr = "datareservoirio.__version__"}
+version = { attr = "datareservoirio.__version__" }
 
 [tool.pytest.ini_options]
 pythonpath = [".", "src"]
@@ -74,4 +72,5 @@ deps =
     sphinx==5.3.0
     pydata-sphinx-theme==0.11.0
     myst_parser<2.0
-"""
+"""
+

tests/conftest.py

@@ -1,4 +1,3 @@
-import logging
 from io import BytesIO
 from pathlib import Path
 from unittest.mock import Mock
@@ -13,12 +12,6 @@
 TEST_PATH = Path(__file__).parent
 
 
-@pytest.fixture(autouse=True)
-def disable_logging(monkeypatch):
-    """Disable logging to Application Insight"""
-    monkeypatch.setattr("datareservoirio.client.AzureLogHandler", logging.NullHandler())
-
-
 @pytest.fixture
 def response_cases():
     class ResponseCaseHandler:

tests/test__logging.py

@@ -3,7 +3,9 @@
 
 import pytest
 
-from datareservoirio._logging import exceptions_logger, log_decorator
+from datareservoirio._logging import get_exceptions_logger, log_decorator
+
+exceptions_logger = get_exceptions_logger()
 
 
 class my_test_class:

tests/test_client.py

@@ -13,11 +13,13 @@
 from tenacity import RetryError
 
 import datareservoirio as drio
-from datareservoirio._logging import exceptions_logger
+from datareservoirio._logging import get_exceptions_logger
 from datareservoirio._utils import DataHandler
 
 TEST_PATH = Path(__file__).parent
 
+exceptions_logger = get_exceptions_logger()
+
 
 def change_logging(self, msg, *args, exc_info=True, **kwargs):
     if kwargs["extra"]: