Ensure sorted data on upload of data from python (#126)

Author: hanne-opseth-rygg-4ss

datareservoirio/client.py

@@ -101,7 +101,7 @@ def create(self, series=None, wait_on_verification=True):
         ----------
         series : pandas.Series, optional
             Series with index (as DatetimeIndex-like or integer array). Default
-            is None.
+            is None. Needs to be sorted on index.
         wait_on_verification : bool (optional)
             All series are subjected to a server-side data validation before
             they are made available for consumption; failing validation will
@@ -127,6 +127,11 @@ def create(self, series=None, wait_on_verification=True):
             response.raise_for_status()
             return response.json()
 
+        if not series.index.is_monotonic_increasing:
+            raise ValueError(
+                "Index not sorted. Please sort series on index before creating a timeseries."
+            )
+
         df = self._verify_and_prepare_series(series)
 
         response_file = self._auth_session.post(
@@ -163,7 +168,7 @@ def append(self, series, series_id, wait_on_verification=True):
         Parameters
         ----------
         series : pandas.Series
-            Series with index (as DatetimeIndex-like or integer array).
+            Series with index (as DatetimeIndex-like or integer array). Needs to be sorted on index.
         series_id : string
             The identifier of the existing series.
         wait_on_verification : bool (optional)
@@ -182,6 +187,10 @@ def append(self, series, series_id, wait_on_verification=True):
         dict
             The response from DataReservoir.io.
         """
+        if not series.index.is_monotonic_increasing:
+            raise ValueError(
+                "Index not sorted. Please sort series on index before appending data."
+            )
         df = self._verify_and_prepare_series(series)
 
         response_file = self._auth_session.post(
@@ -411,7 +420,21 @@ def get(
         else:
             df = pd.DataFrame(columns=("index", "values")).astype({"index": "int64"})
 
-        series = df.set_index("index").squeeze("columns").loc[start:end].copy(deep=True)
+        try:
+            series = (
+                df.set_index("index").squeeze("columns").loc[start:end].copy(deep=True)
+            )
+        except KeyError as e:
+            logging.warning(
+                "The time series you requested is not properly ordered. The data will be sorted to attempt to resolve the issue. Please note that this operation may take some time."
+            )
+            series = (
+                df.set_index("index")
+                .sort_index()
+                .squeeze("columns")
+                .loc[start:end]
+                .copy(deep=True)
+            )
         series.index.name = None
 
         if series.empty and raise_empty:  # may become empty after slicing

docs/user_guide/index.rst

@@ -38,8 +38,8 @@ explained below.
 Series
 ------
 A series is a one-dimensional sequence with numeric values (64-bit float) and
-unique indicies (64-bit integer). (Consequently, each numeric value is natively
-represented with 128-bits.) Each series is assigned a unique identifier
+unique indices (64-bit integer). Consequently, each numeric value is natively
+represented with 128-bits. Each series is assigned a unique identifier
 `TimeSeriesId` (guid) for convenient access. Furthermore, a series can be
 enriched with :ref:`metadata <metadata>`.
 
@@ -76,8 +76,8 @@ But, that responsibility is left to the user/app/service that uses
 
 Metadata entries are organized using ``namespace`` and ``key``. A ``namespace``
 can be thought of as a table and ``key`` is the row index. Then a row can have
-any number of arbitrary number of columns. (Note that rows in a table do not 
-have to share the columns!). This resembles "table storage" paradigm for those
+any number of arbitrary number of columns (Note that rows in a table do not 
+have to share the columns). This resembles "table storage" paradigm for those
 who are familiar with that.
 
 Thus, a ``namespace`` and ``key`` combination uniquely defines a metadata

docs/user_guide/manage_series.rst

@@ -48,8 +48,8 @@ information is returned:
     converted to UTC and therefore, time zone information is lost when data is
     stored in `DataReservoir.io`_.
 
-You can also store a sequence of data. However, you are required to define an
-integer index. (This is useful when appending and updating the data later.)
+You can also store a sequence of data. However, you are required to define an increasing
+integer index. This is useful when appending and updating the data later.
 
 Store sequence:
 
@@ -75,6 +75,9 @@ data:
     series_id = response['TimeSeriesId']
     response = client.append(series, series_id)
 
+.. important::
+
+    The index of the series must be sorted. This is also efficient when accessing the data later.
 
 Data verification process
 -------------------------

tests/test_client.py

@@ -4,6 +4,7 @@
 import types
 from encodings.utf_8 import encode
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
 import pandas as pd
 import pytest
@@ -58,6 +59,10 @@ def fail_with_invalid_json_error(self, url, timeout):
         raise InvalidJSONError()
 
 
+def _mock_blob_sequence_days(response_json):
+    return {1: "file1", 2: "file2"}
+
+
 class Test_Client:
     """
     Tests the ``datareservoirio.Client`` class.
@@ -158,6 +163,53 @@ def test_get_raise_empty(self, client):
         with pytest.raises(ValueError):
             client.get("e3d82cda-4737-4af9-8d17-d9dfda8703d0", raise_empty=True)
 
+    def test_get_keyerror(self, client):
+        series_id = "test_series_id"
+        start = "2023-01-01"
+        end = "2023-01-02"
+
+        response_mock = MagicMock()
+        response_mock.status_code = 200
+        response_mock.json.return_value = {
+            "Files": [
+                {"Chunks": "file1"},
+                {"Chunks": "file2"},
+            ]  # mock files with correct structure
+        }
+
+        client._auth_session.get = MagicMock(return_value=response_mock)
+
+        def mock_storage_get(blob_sequence_i):
+            if blob_sequence_i == "file1":
+                return pd.DataFrame(
+                    {
+                        "index": [1672358410000000000, 1672358400000000000],
+                        "values": [100, 200],
+                    }
+                )
+            elif blob_sequence_i == "file2":
+                return pd.DataFrame(
+                    {
+                        "index": [1672358410000000000, 1672358420000000000],
+                        "values": [200, 400],
+                    }
+                )
+            else:
+                raise ValueError("Unexpected blob_sequence_i value")
+
+        client._storage.get = MagicMock(side_effect=mock_storage_get)
+
+        with patch(
+            "datareservoirio.client._blob_sequence_days",
+            side_effect=_mock_blob_sequence_days,
+        ):
+            with patch("datareservoirio.logging.warning") as mock_logging_warning:
+                result = client.get(series_id, start, end)
+                mock_logging_warning.assert_called_once_with(
+                    "The time series you requested is not properly ordered. The data will be sorted to attempt to resolve the issue. Please note that this operation may take some time."
+                )
+                assert isinstance(result, pd.Series)
+
     def test_get_raises_end_not_after_start(self, client):
         start = 1672358400000000000
         end = start - 1
@@ -427,6 +479,22 @@ def test_create_upload_raises(self, client, data_float, response_cases):
         with pytest.raises(HTTPError):
             client.create(series=data_float.as_series(), wait_on_verification=True)
 
+    def test_create_raises_valueerror_unsorted_index(self, client):
+        data = pd.Series(
+            [1, 2, 3],
+            index=[
+                pd.to_datetime("2022-04-04"),
+                pd.to_datetime("2022-04-03"),
+                pd.to_datetime("2022-04-05"),
+            ],
+        )
+        with pytest.raises(ValueError) as e:
+            client.create(data)
+        assert (
+            str(e.value)
+            == "Index not sorted. Please sort series on index before creating a timeseries."
+        )
+
     def test_append(
         self, client, data_float, mock_requests, bytesio_with_memory, response_cases
     ):
@@ -501,6 +569,23 @@ def test_append_upload_raises(self, client, data_float, response_cases):
         with pytest.raises(HTTPError):
             client.append(data_float.as_series(), series_id, wait_on_verification=True)
 
+    def test_append_raises_valueerror_unsorted_index(self, client):
+        series_id = "d30519af-5035-4093-a425-dafd857ad0ef"
+        data = pd.Series(
+            [1, 2, 3],
+            index=[
+                pd.to_datetime("2022-04-04"),
+                pd.to_datetime("2022-04-03"),
+                pd.to_datetime("2022-04-05"),
+            ],
+        )
+        with pytest.raises(ValueError) as e:
+            client.append(data, series_id)
+        assert (
+            str(e.value)
+            == "Index not sorted. Please sort series on index before appending data."
+        )
+
     @pytest.mark.parametrize("data", ("data_float", "data_string"))
     def test__verify_and_prepare_series(self, client, data, request):
         data = request.getfixturevalue(data)