update to allow override file system driver using py fs

tutankalex · tutankalex · commit 588bb110118b · 2023-07-18T17:00:48.000+08:00
the architecture is jank because it's set at the class level.
should be made more pure later.
also update github links + nbdev dep
diff --git a/nbs/00_core.ipynb b/nbs/00_core.ipynb
@@ -35,30 +35,13 @@
     "import logging\n",
     "from jsonschema import validate, ValidationError\n",
     "from typing import Union\n",
+    "from fs.base import FS\n",
+    "from fs.osfs import OSFS\n",
     "\n",
     "\n",
     "logger = logging.getLogger(__name__)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#| export\n",
-    "def load_json(json_source: Union[str, dict]=None) -> dict:\n",
-    "    '''\n",
-    "    convenience method to return a dict from either\n",
-    "    a file path or an already-loaded dict\n",
-    "    '''\n",
-    "    if isinstance(json_source, str):\n",
-    "        with open(json_source) as ifile:\n",
-    "            return json.load(ifile)\n",
-    "    elif isinstance(json_source, dict):\n",
-    "        return json_source"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -232,19 +215,33 @@
     "\n",
     "\n",
     "class ConfigValidator(object):\n",
+    "\n",
+    "    DEFAULT_STORAGE_DRIVER: FS = OSFS('.')\n",
     "    \n",
     "    CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME = 'CONFIG_VALIDATOR_JSON_SCHEMA'\n",
-    "    \n",
+    "\n",
+    "    @classmethod\n",
+    "    def load_json(cls, json_source: Union[str, dict]=None) -> dict:\n",
+    "        '''\n",
+    "        convenience method to return a dict from either\n",
+    "        a file path or an already-loaded dict\n",
+    "        '''\n",
+    "        if isinstance(json_source, str):\n",
+    "            with cls.DEFAULT_STORAGE_DRIVER.open(json_source) as ifile:\n",
+    "                return json.load(ifile)\n",
+    "        elif isinstance(json_source, dict):\n",
+    "            return json_source\n",
+    "\n",
     "    @classmethod\n",
     "    def get_default_json_schema(cls):\n",
     "        if cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME in os.environ:\n",
     "            expected_json_schema_path = \\\n",
     "                os.environ[cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME]\n",
-    "            with open(expected_json_schema_path) as ifile:\n",
+    "            with cls.DEFAULT_STORAGE_DRIVER.open(expected_json_schema_path) as ifile:\n",
     "                return json.load(ifile)\n",
     "        return None\n",
     "\n",
-    "    def __init__(self, json_schema: Union[str, dict]=None):\n",
+    "    def __init__(self, json_schema: Union[str, dict]=None, storage_driver: FS=None):\n",
     "        '''\n",
     "        :param json_schema: a str path to a json schema file, or a schema in dict form\n",
     "        \n",
@@ -253,8 +250,9 @@
     "        `CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME`\n",
     "        to find a JSON schema file\n",
     "        '''\n",
+    "        self.storage_driver = storage_driver or self.__class__.DEFAULT_STORAGE_DRIVER\n",
     "        if isinstance(json_schema, (str, dict)):\n",
-    "            self._json_schema = load_json(json_schema)\n",
+    "            self._json_schema = self.__class__.load_json(json_schema)\n",
     "        elif (default_schema := self.__class__.get_default_json_schema()):\n",
     "            self._json_schema = default_schema\n",
     "        else:\n",
@@ -347,7 +345,17 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "$:\t'string_value_with_enum' is a required property\n",
+      "$:\t'MY_INTEGER_VALUE' is a required property\n",
+      "$:\t'A_NUMERIC_VALUE' is a required property\n"
+     ]
+    }
+   ],
    "source": [
     "#| hide\n",
     "test_fail(ConfigValidator.load_validated_config, args=(example_properties_schema, {}))"
@@ -395,7 +403,15 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "$.string_value_with_enum:\t'blah-blah' is not one of ['it', 'can', 'only', 'be', 'one', 'of', 'these']\n"
+     ]
+    }
+   ],
    "source": [
     "#| hide\n",
     "test_fail(ConfigValidator.load_validated_config,\n",
@@ -411,7 +427,15 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "$.MY_INTEGER_VALUE:\t'5555.999' is not of type 'integer'\n"
+     ]
+    }
+   ],
    "source": [
     "#| hide\n",
     "test_fail(ConfigValidator.load_validated_config,\n",
@@ -427,7 +451,15 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "$.A_NUMERIC_VALUE:\t'WHAT???' is not of type 'number'\n"
+     ]
+    }
+   ],
    "source": [
     "#| hide\n",
     "test_fail(ConfigValidator.load_validated_config,\n",
@@ -443,7 +475,15 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "$.A_NUMERIC_VALUE:\t13.0 is less than the minimum of 22\n"
+     ]
+    }
+   ],
    "source": [
     "#| hide\n",
     "test_fail(ConfigValidator.load_validated_config,\n",
@@ -455,6 +495,40 @@
     "}))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| hide\n",
+    "# FIXME this test clobbers the environment\n",
+    "# FIXME this is a code smell on the class design\n",
+    "# here we're testing the ability to override the storage driver (memoryfs here)\n",
+    "\n",
+    "from fs.memoryfs import MemoryFS\n",
+    "memfs = MemoryFS()\n",
+    "\n",
+    "with memfs.open('schema.json', 'w') as ofile:\n",
+    "    ofile.write(json.dumps(example_properties_schema))\n",
+    "    os.environ['CONFIG_VALIDATOR_JSON_SCHEMA'] = ofile.name\n",
+    "\n",
+    "ConfigValidator.DEFAULT_STORAGE_DRIVER = memfs\n",
+    "validator = ConfigValidator()\n",
+    "validated_config = validator.load_config({\n",
+    "    'string_value_with_enum': 'these',\n",
+    "    'MY_INTEGER_VALUE': '-85',\n",
+    "    'A_NUMERIC_VALUE': '1.23e4',\n",
+    "})\n",
+    "\n",
+    "test_eq(validated_config, {\n",
+    "    'string_value_with_enum': 'these',\n",
+    "    'MY_INTEGER_VALUE': -85,\n",
+    "    'A_NUMERIC_VALUE': 12300.0,\n",
+    "    '_____A_STRING_VALUE____with_default__': 'underscores_and spaces',\n",
+    "})"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/nbs/01_cli.ipynb b/nbs/01_cli.ipynb
@@ -31,7 +31,6 @@
     "from schematized_config.core import (\n",
     "    ConfigValidator,\n",
     "    ConfigValidatorException,\n",
-    "    load_json,\n",
     "    extract_declared_items,\n",
     ")\n",
     "\n",
@@ -72,7 +71,7 @@
    "source": [
     "#| export\n",
     "def generate_sample_dotenv(json_schema: Union[str, dict], seed_config: dict=None):\n",
-    "    schema_dict = load_json(json_schema)\n",
+    "    schema_dict = ConfigValidator.load_json(json_schema)\n",
     "    merged_config = dict(os.environ)\n",
     "    default_dotenv = dotenv.dotenv_values()\n",
     "    merged_config.update(default_dotenv)\n",
diff --git a/nbs/nbdev.yml b/nbs/nbdev.yml
@@ -3,7 +3,7 @@ project:
 
 website:
   title: "python-schematized-config"
-  site-url: "https://aistriketeam.github.io/python-schematized-config"
+  site-url: "https://tutankalex.github.io/python-schematized-config"
   description: "validate configs using json schema"
   repo-branch: main
-  repo-url: "https://github.com/aistriketeam/python-schematized-config"
+  repo-url: "https://github.com/tutankalex/python-schematized-config"
diff --git a/schematized_config/__init__.py b/schematized_config/__init__.py
@@ -1 +1 @@
-__version__ = "0.0.1"
+__version__ = "0.0.2"
diff --git a/schematized_config/_modidx.py b/schematized_config/_modidx.py
@@ -2,8 +2,8 @@
 
 d = { 'settings': { 'branch': 'main',
                 'doc_baseurl': '/python-schematized-config',
-                'doc_host': 'https://aistriketeam.github.io',
-                'git_url': 'https://github.com/aistriketeam/python-schematized-config',
+                'doc_host': 'https://tutankalex.github.io',
+                'git_url': 'https://github.com/tutankalex/python-schematized-config',
                 'lib_path': 'schematized_config'},
   'syms': { 'schematized_config.cli': { 'schematized_config.cli.generate_sample_dotenv': ( 'cli.html#generate_sample_dotenv',
                                                                                            'schematized_config/cli.py'),
@@ -19,6 +19,8 @@
                                                                                                   'schematized_config/core.py'),
                                          'schematized_config.core.ConfigValidator.load_dotenv': ( 'core.html#configvalidator.load_dotenv',
                                                                                                   'schematized_config/core.py'),
+                                         'schematized_config.core.ConfigValidator.load_json': ( 'core.html#configvalidator.load_json',
+                                                                                                'schematized_config/core.py'),
                                          'schematized_config.core.ConfigValidator.load_validated_config': ( 'core.html#configvalidator.load_validated_config',
                                                                                                             'schematized_config/core.py'),
                                          'schematized_config.core.ConfigValidator.load_validated_environment': ( 'core.html#configvalidator.load_validated_environment',
@@ -30,5 +32,4 @@
                                          'schematized_config.core.coerce_primitive_values': ( 'core.html#coerce_primitive_values',
                                                                                               'schematized_config/core.py'),
                                          'schematized_config.core.extract_declared_items': ( 'core.html#extract_declared_items',
-                                                                                             'schematized_config/core.py'),
-                                         'schematized_config.core.load_json': ('core.html#load_json', 'schematized_config/core.py')}}}
+                                                                                             'schematized_config/core.py')}}}
diff --git a/schematized_config/cli.py b/schematized_config/cli.py
@@ -7,7 +7,6 @@
 from schematized_config.core import (
     ConfigValidator,
     ConfigValidatorException,
-    load_json,
     extract_declared_items,
 )
 
@@ -32,7 +31,7 @@ def validate_env(json_schema: Union[str, dict], dotenv_path: str=None):
 
 # %% ../nbs/01_cli.ipynb 4
 def generate_sample_dotenv(json_schema: Union[str, dict], seed_config: dict=None):
-    schema_dict = load_json(json_schema)
+    schema_dict = ConfigValidator.load_json(json_schema)
     merged_config = dict(os.environ)
     default_dotenv = dotenv.dotenv_values()
     merged_config.update(default_dotenv)
diff --git a/schematized_config/core.py b/schematized_config/core.py
@@ -1,8 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
 
 # %% auto 0
-__all__ = ['logger', 'load_json', 'coerce_primitive_values', 'extract_declared_items', 'ConfigValidatorException',
-           'ConfigValidator']
+__all__ = ['logger', 'coerce_primitive_values', 'extract_declared_items', 'ConfigValidatorException', 'ConfigValidator']
 
 # %% ../nbs/00_core.ipynb 2
 from nbdev.showdoc import *
@@ -15,23 +14,13 @@
 import logging
 from jsonschema import validate, ValidationError
 from typing import Union
+from fs.base import FS
+from fs.osfs import OSFS
 
 
 logger = logging.getLogger(__name__)
 
 # %% ../nbs/00_core.ipynb 3
-def load_json(json_source: Union[str, dict]=None) -> dict:
-    '''
-    convenience method to return a dict from either
-    a file path or an already-loaded dict
-    '''
-    if isinstance(json_source, str):
-        with open(json_source) as ifile:
-            return json.load(ifile)
-    elif isinstance(json_source, dict):
-        return json_source
-
-# %% ../nbs/00_core.ipynb 4
 def coerce_primitive_values(json_schema: dict, data: dict) -> dict:
     '''
     given a JSON schema dict, return a dict where the values that have
@@ -73,7 +62,7 @@ def coerce_primitive_values(json_schema: dict, data: dict) -> dict:
             continue
     return out
 
-# %% ../nbs/00_core.ipynb 7
+# %% ../nbs/00_core.ipynb 6
 def extract_declared_items(json_schema: dict, data: dict) -> dict:
     '''
     given a JSON schema dict, return a dict where
@@ -94,7 +83,7 @@ def extract_declared_items(json_schema: dict, data: dict) -> dict:
             out[required_property] = property_schema['default']
     return out
 
-# %% ../nbs/00_core.ipynb 9
+# %% ../nbs/00_core.ipynb 8
 class ConfigValidatorException(Exception):
     
     def __init__(self, errors):
@@ -103,19 +92,33 @@ def __init__(self, errors):
 
 
 class ConfigValidator(object):
+
+    DEFAULT_STORAGE_DRIVER: FS = OSFS('.')
     
     CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME = 'CONFIG_VALIDATOR_JSON_SCHEMA'
-    
+
+    @classmethod
+    def load_json(cls, json_source: Union[str, dict]=None) -> dict:
+        '''
+        convenience method to return a dict from either
+        a file path or an already-loaded dict
+        '''
+        if isinstance(json_source, str):
+            with cls.DEFAULT_STORAGE_DRIVER.open(json_source) as ifile:
+                return json.load(ifile)
+        elif isinstance(json_source, dict):
+            return json_source
+
     @classmethod
     def get_default_json_schema(cls):
         if cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME in os.environ:
             expected_json_schema_path = \
                 os.environ[cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME]
-            with open(expected_json_schema_path) as ifile:
+            with cls.DEFAULT_STORAGE_DRIVER.open(expected_json_schema_path) as ifile:
                 return json.load(ifile)
         return None
 
-    def __init__(self, json_schema: Union[str, dict]=None):
+    def __init__(self, json_schema: Union[str, dict]=None, storage_driver: FS=None):
         '''
         :param json_schema: a str path to a json schema file, or a schema in dict form
         
@@ -124,8 +127,9 @@ def __init__(self, json_schema: Union[str, dict]=None):
         `CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME`
         to find a JSON schema file
         '''
+        self.storage_driver = storage_driver or self.__class__.DEFAULT_STORAGE_DRIVER
         if isinstance(json_schema, (str, dict)):
-            self._json_schema = load_json(json_schema)
+            self._json_schema = self.__class__.load_json(json_schema)
         elif (default_schema := self.__class__.get_default_json_schema()):
             self._json_schema = default_schema
         else:
diff --git a/settings.ini b/settings.ini

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.0.1"`
	`1`	`+__version__ = "0.0.2"`