Merge branch 'main' into fix-lang-tags

Author: hrshdhgd

docs/conf.py

@@ -7,13 +7,15 @@
 import re
 import sys
 from datetime import date
-from rdf-sql-bulkloader import __version__
+
+from rdf_sql_bulkloader import __version__
+
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-project = 'rdf-sql-bulkloader'
+project = "rdf-sql-bulkloader"
 copyright = 'f"{date.today().year}, Chris Mungall <cjmungall@lbl.gov>'
-author = 'Chris Mungall <cjmungall@lbl.gov>'
+author = "Chris Mungall <cjmungall@lbl.gov>"
 release = __version__
 
 # -- General configuration ---------------------------------------------------
@@ -45,13 +47,13 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'sphinx_rtd_theme'
-html_static_path = ['_static']
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.

src/rdf_sql_bulkloader/__init__.py

@@ -3,4 +3,4 @@
 
 __version__ = metadata.version(__name__)
 
-from rdf_sql_bulkloader.loaders.sqlite3_bulkloader import SqliteBulkloader
+from rdf_sql_bulkloader.loaders.sqlite3_bulkloader import SqliteBulkloader

src/rdf_sql_bulkloader/cli.py

@@ -1,13 +1,14 @@
 """Command line interface for rdf-sql-bulkloader."""
+import logging
 from pathlib import Path
 
 import click
-import logging
 
-from rdf_sql_bulkloader import __version__, SqliteBulkloader
+from rdf_sql_bulkloader import SqliteBulkloader, __version__
 
 logger = logging.getLogger(__name__)
 
+
 @click.group()
 @click.option("-v", "--verbose", count=True)
 @click.option("-q", "--quiet")
@@ -30,10 +31,9 @@ def main(verbose: int, quiet: bool):
 
 @main.command()
 @click.option("--output", "-o", required=True)
-@click.option("--force/--no-force",
-              default=False,
-              show_default=True,
-              help="Recreates db if already present")
+@click.option(
+    "--force/--no-force", default=False, show_default=True, help="Recreates db if already present"
+)
 @click.argument("files", nargs=-1)
 def load_sqlite(files, output, force: bool):
     """Run the rdf-sql-bulkloader's demo command."""
@@ -45,13 +45,11 @@ def load_sqlite(files, output, force: bool):
             raise ValueError(f"Path exists {output_path}")
     loader = SqliteBulkloader(output)
     if len(files) > 1:
-        logging.warning(f"Blank nodes may be shared TODO FIX ME")
+        logging.warning("Blank nodes may be shared TODO FIX ME")
     for file in files:
         print(file)
         loader.bulkload(file)
 
-     
-
 
 if __name__ == "__main__":
     main()

src/rdf_sql_bulkloader/loaders/__init__.py

@@ -0,0 +1,5 @@
+"""
+loaders
+
+Currently only SQLite3 is supported
+"""

src/rdf_sql_bulkloader/loaders/bulkloader.py

@@ -1,11 +1,13 @@
+"""
+Base class for loaders
+"""
 import re
 from abc import ABC
 from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Union, Tuple, Iterator, Mapping, Optional, List
+from typing import Iterator, List, Mapping, Optional, Tuple, Union
 
-import curies
 import lightrdf
 from curies import Converter
 from prefixmaps.io.parser import load_multi_context
@@ -16,16 +18,17 @@
 re_blank_node = re.compile(r'^riog(\d+)$')
 
 
+
 DDL = """
 CREATE TABLE statement (
-	id TEXT,
-	subject TEXT,
-	predicate TEXT,
-	object TEXT,
-	value TEXT,
-	datatype TEXT,
-	language TEXT,
-        graph TEXT
+    id TEXT,
+    subject TEXT,
+    predicate TEXT,
+    object TEXT,
+    value TEXT,
+    datatype TEXT,
+    language TEXT,
+    graph TEXT
 );
 """
 
@@ -70,22 +73,23 @@ def _parse_literal(o: str) -> Tuple[OBJECT_VALUE, OBJECT_DATATYPE, OBJECT_LANG]:
                 raise ValueError(f"Cannot parse {o}")
     return o_value, o_datatype, o_lang
 
+
 def _parse_literal_as_value(o: str) -> str:
     return _parse_literal(o)[0]
 
 
-
 @dataclass
 class BulkLoader(ABC):
     """
     Base class for all bulk loaders
     """
+
     path: str
     named_prefix_maps: List[str] = None
     prefix_map: PREFIX_MAP = None
     converter: Converter = None
     index_statements = False
-    
+
     def __post_init__(self):
         if self.prefix_map is None:
             named_prefix_maps = self.named_prefix_maps
@@ -101,7 +105,7 @@ def _set_converter(self):
             self.converter = Converter.from_prefix_map(self.prefix_map)
 
     def bulkload(self, path: str):
-        raise NotImplemented
+        raise NotImplementedError
 
     def _parse_node(self, o: str) -> str:
         if re_blank_node.match(o):
@@ -131,9 +135,9 @@ def statements(self, path: Union[Path, str]) -> Iterator[STATEMENT]:
         prefix_node_map = defaultdict(lambda: [None, None])
         statement_node_map = defaultdict(lambda: [None, None])
         for s, p, o in doc.search_triples(None, None, None):
-            if p == 'http://www.w3.org/ns/shacl#prefix':
+            if p == "http://www.w3.org/ns/shacl#prefix":
                 prefix_node_map[s][0] = _parse_literal_as_value(o)
-            elif p == 'http://www.w3.org/ns/shacl#namespace':
+            elif p == "http://www.w3.org/ns/shacl#namespace":
                 prefix_node_map[s][1] = _parse_literal_as_value(o)
             elif self.index_statements:
                 # this is optional as it may be inefficient to do at the python level

tests/__init__.py

@@ -1,9 +1,7 @@
 import os
 from pathlib import Path
 
-TEST_PREFIX_MAP = {
-    "GO": "http://purl.obolibrary.org/obo/GO_"
-}
+TEST_PREFIX_MAP = {"GO": "http://purl.obolibrary.org/obo/GO_"}
 
 ROOT = os.path.abspath(os.path.dirname(__file__))
 INPUT_DIR = Path(ROOT) / "input"
@@ -15,6 +13,7 @@
 TEST_LANG_INPUT_OWL = INPUT_DIR / f"{TEST_LANG_BASE}.owl"
 TEST_LANG_INPUT_TTL = INPUT_DIR / f"{TEST_LANG_BASE}.ttl"
 
+
 def output_path(fn: str) -> str:
     return str(Path(OUTPUT_DIR) / fn)
 

tests/test_sqlite3_bulkloader.py

@@ -3,14 +3,34 @@
 import unittest
 
 from rdf_sql_bulkloader import SqliteBulkloader
+
 from tests import TEST_INPUT_OWL, NUCLEUS, TEST_PREFIX_MAP, TEST_LANG_INPUT_OWL, TEST_LANG_INPUT_TTL
 
-CASES =[
-    (None, 'GO:0005634', 'RO:0002161', 'NCBITaxon:2', None, None, None, None),
-    (None, 'GO:0005634', 'IAO:0000115', None, "A membrane-bounded organelle of eukaryotic cells in which chromosomes are housed and replicated. In most cells, the nucleus contains all of the cell's chromosomes except the organellar chromosomes, and is the site of RNA synthesis and processing. In some species, or in specialized cell types, RNA metabolism or DNA replication may be absent.", 'http://www.w3.org/2001/XMLSchema#string', None, None),
-    (None, 'GO:0005634', 'rdf:type', 'owl:Class', None, None, None, None),
+DEFN = (
+    "A membrane-bounded organelle of eukaryotic cells in which chromosomes "
+    + "are housed and replicated. In most cells, the nucleus contains all of the cell's "
+    + "chromosomes except the organellar chromosomes, and is the site of RNA synthesis "
+    + "and processing. In some species, or in specialized cell types, "
+    + "RNA metabolism or DNA replication may be absent."
+)
+
+
+CASES = [
+    (None, "GO:0005634", "RO:0002161", "NCBITaxon:2", None, None, None, None),
+    (
+        None,
+        "GO:0005634",
+        "IAO:0000115",
+        None,
+        DEFN,
+        "http://www.w3.org/2001/XMLSchema#string",
+        None,
+        None,
+    ),
+    (None, "GO:0005634", "rdf:type", "owl:Class", None, None, None, None),
 ]
 
+
 class TestSqlit3BulkLoader(unittest.TestCase):
     """Test sqlite3."""
 
@@ -47,3 +67,4 @@ def test_lang_tags_ttl(self):
             loader.bulkload(TEST_LANG_INPUT_TTL)
 
 
+

tox.ini

@@ -4,64 +4,31 @@
 # and then run "tox" from this directory.
 
 [tox]
-# To use a PEP 517 build-backend you are required to configure tox to use an isolated_build:
-# https://tox.readthedocs.io/en/latest/example/package.html
-isolated_build = True
-skipsdist = True
-
+isolated_build = true
 envlist =
-    # always keep coverage-clean first
-    coverage-clean
     lint
     flake8
-    mypy
-    docstr-coverage
-    py
-
-[testenv]
-whitelist_externals =
-    poetry
-commands =
-    poetry run pytest
-description = Run unit tests with pytest. This is a special environment that does not get a name, and
-              can be referenced with "py".
-
-[testenv:coverage-clean]
-deps = coverage
-skip_install = true
-commands = coverage erase
 
 [testenv:lint]
 deps =
     black
     isort
 skip_install = true
 commands =
-    black src/ tests/
-    isort src/ tests/
+    black .
+    isort .
 description = Run linters.
 
-[testenv:doclint]
-deps =
-    rstfmt
-skip_install = true
-commands =
-    rstfmt docs/source/
-description = Run documentation linters.
-
 [testenv:flake8]
 skip_install = true
 deps =
-    darglint
     flake8<5.0.0
-    flake8-black
     flake8-bandit
+    flake8-black
     flake8-bugbear
     flake8-colors
-    flake8-docstrings
     flake8-isort
-    flake8-print
-    pydocstyle
+    pep8-naming
 commands =
     flake8 src/ tests/
 description = Run the flake8 tool with several plugins (bandit, docstrings, import order, pep8 naming).
@@ -72,22 +39,29 @@ description = Run the flake8 tool with several plugins (bandit, docstrings, impo
 #########################
 [flake8]
 ignore =
-    DAR101 # Missing parameter(s) in Docstring: - with_git_hash
-    DAR201 # Missing "Returns" in Docstring: - return
-    DAR301 # Missing "Yields" in Docstring: - yield
-    E111 # indentation is not a multiple of 4
-    T201 # print found.
-
-[testenv:mypy]
-deps = mypy
-skip_install = true
-commands = mypy --install-types --non-interactive --ignore-missing-imports src/
-description = Run the mypy tool to check static typing on the project.
+    E203
+    W503
+    C901 # needs code change so ignoring for now.
+    E731 # needs code change so ignoring for now.
+    S101 # asserts are fine
+    S106 # flags false positives with test_table_filler
+    N801 # mixed case is bad but there's a lot of auto-generated code
+    N815 # same ^
+    S404 # Consider possible security implications associated with the subprocess module.
+    S108 # Probable insecure usage of temp file/directory.
+    S307 # Use of possibly insecure function - consider using safer ast.literal_eval.
+    S603 # subprocess call - check for execution of untrusted input.
+    S607 # Starting a process with a partial executable path ["open" in both cases]
+    S608 # Possible SQL injection vector through string-based query construction.
+    B024 # StreamingWriter is an abstract base class, but it has no abstract methods. 
+         # Remember to use @abstractmethod, @abstractclassmethod and/or @abstractproperty decorators.
+per-file-ignores = __init__.py:F401
 
-[testenv:docstr-coverage]
-skip_install = true
-deps =
-    docstr-coverage
-commands =
-    docstr-coverage src/ tests/ --skip-private --skip-magic
-description = Run the docstr-coverage tool to check documentation coverage
+max-line-length = 120
+max-complexity = 13
+import-order-style = pycharm
+application-import-names =
+    oaklib
+    tests
+exclude =
+    datamodels   ## datamodels are auto-generated