Merge pull request #276 from jacebrowning/delete-invalid-yaml

Delete unparseable YAML files in manager methods

Author: jacebrowning

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Release Notes
 
+## 1.4.3 (2022-08-19)
+
+- Fixed handling of unparseable YAML files in manager methods.
+
 ## 1.4.2 (2022-07-29)
 
 - Fixed exception when `Union[int, float]` or `Union[str, ...]` is used.

datafiles/manager.py

@@ -12,6 +12,7 @@
 
 import log
 from parse import parse
+from ruamel.yaml.parser import ParserError
 
 from . import hooks
 
@@ -40,7 +41,12 @@ def get(self, *args, **kwargs) -> Model:
 
         with hooks.disabled():
             instance = self.model(*args, **kwargs)
-            instance.datafile.load()
+            try:
+                instance.datafile.load()
+            except ParserError:
+                log.critical(f"Deleting unparseable YAML: {instance.datafile.path}")
+                instance.datafile.path.unlink()
+                instance.datafile.load()
 
         return instance
 

datafiles/tests/test_manager.py

@@ -1,7 +1,9 @@
 # pylint: disable=unused-variable
 
 import os
+import shutil
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Optional
 from unittest.mock import patch
 
@@ -26,6 +28,7 @@ class MyClass:
 def describe_manager():
     @pytest.fixture
     def manager():
+        shutil.rmtree(Path(__file__).parent / "files", ignore_errors=True)
         model = create_model(MyClass, pattern="files/{self.foo}.yml")
         return Manager(model)
 
@@ -46,6 +49,12 @@ def when_file_exists(mock_load, expect, manager):
         def when_file_missing(expect, manager):
             expect(manager.get_or_none(foo=3, bar=4)).is_(None)
 
+        def when_file_corrupt(expect, manager):
+            instance = manager.get_or_create(foo=2, bar=1)
+            instance.datafile.path.write_text("{")
+            instance = manager.get_or_none(foo=2, bar=2)
+            expect(instance).is_(None)
+
     def describe_get_or_create():
         @patch("datafiles.mapper.Mapper.save")
         @patch("datafiles.mapper.Mapper.load")
@@ -64,6 +73,12 @@ def when_file_missing(mock_save, mock_load, expect, manager):
             expect(mock_save.called).is_(True)
             expect(mock_load.called).is_(True)
 
+        def when_file_corrupt(expect, manager):
+            instance = manager.get_or_create(foo=2, bar=1)
+            instance.datafile.path.write_text("{")
+            instance = manager.get_or_create(foo=2, bar=2)
+            expect(instance.bar) == 2
+
     def describe_all():
         @patch("datafiles.mapper.Mapper.exists", False)
         def when_no_files_exist(expect, manager):

docs/requirements.txt

@@ -1,4 +1,5 @@
 jinja2==3.0.3; python_version >= "3.6"
 markdown==3.3.4; python_version >= "3.6"
 mkdocs==1.3.1; python_version >= "3.6"
+jupyterlab-pygments==0.2.2; python_version >= "3.7"
 pygments==2.10.0; python_version >= "3.5"

poetry.lock

@@ -1,7 +1,7 @@
 [tool.poetry]
 
 name = "datafiles"
-version = "1.4.2"
+version = "1.4.3"
 description = "File-based ORM for dataclasses."
 
 license = "MIT"