Style check directory case insensitive (#90)

* Use a git log command that works on git 1.8.x and 2.x

* Use a git log command that works on git 1.8.x and 2.x

* Use a git log command that works on git 1.8.x and 2.x

* Use a git log command that works on git 1.8.x and 2.x

* Use a git log command that works on git 1.8.x and 2.x

* Add type check fixes.

* Add type check fixes.

* build fix

* build fix

* build fix

* Add release tagging

* Add release tagging

Minor updates

* Add release tagging

* Add release tagging

* Bandit checks

* Git tagging fix

* Git tagging fix

* Git tagging fix

* Make tagging work if ssh is setup by the screwdriver server setup.

* Support the old package publish environment setting variable so old templates don't break.

* Don't run the validation on centos 5 since it doesn't work.

* Don't run the validation on centos 5 since it doesn't work.

* Remove debug print

* Don't override the package version

* Display a message when changing the metadata in screwdriver.

* - Make the release tagger use the Versioner class to get the metadata version so there aren't
multiple implementations of the same thing.
- Update the Versioner base class to allow specifying the meta command as an argument.  To simplify writing tests.

* Don't output the bytestring as output but the string instead

* Don't run the validation on centos 5 since it doesn't work.

* Type validation fix and codestyle fix

* Don't run the validation on centos 5 since it doesn't work.

* Don't run the validation on centos 5 since it doesn't work.

* Don't run the validation on centos 5 since it doesn't work.

Author: dwighthubbard

setup.cfg

@@ -86,6 +86,7 @@ screwdrivercd.documentation.plugin =
 [options.extras_require]
 documentation = 
 	dhubbard-sphinx-markdown-tables
+	markdown<3.2    # This is only needed for the mkdocs-material package to install
 	guzzle_sphinx_theme
 	markdown-include
 	mkdocs-bootstrap4
@@ -98,6 +99,7 @@ documentation =
 	sphinx_rtd_theme
 doc_build = 
 	dhubbard-sphinx-markdown-tables
+	markdown<3.2    # This is only needed for the mkdocs-material package to install
 	guzzle_sphinx_theme
 	markdown-include
 	mkdocs-bootstrap4

src/screwdrivercd/documentation/exceptions.py

@@ -5,7 +5,9 @@
 """
 class DocBuildError(Exception):
     """Generic Documentation Generation Error"""
+    plugin: str=''
 
 
 class DocPublishError(Exception):
     """Documentation Publication Error"""
+    plugin: str=''

src/screwdrivercd/documentation/plugin.py

@@ -291,7 +291,7 @@ def build_documentation(self) -> str:
 
         return self.build_dest
 
-    def publish_documentation(self, clear_before_build=True):  # pragma: no cover
+    def publish_documentation(self, clear_before_build=True, push=True):  # pragma: no cover
         """
         Build and publsh the documentation
 
@@ -309,12 +309,15 @@ def publish_documentation(self, clear_before_build=True):  # pragma: no cover
         # Build the documentation
         self.build_documentation()
 
+        if not push:
+            return
+
         # Make sure there is a clone url before trying to publish
-        if not self.clone_url:
+        if not self.clone_url:  # pragma: no cover
             raise DocPublishError(f'Unable to determine a valid git clone url to publish to')
 
         # Checkout the documentation branch into a temporary directory, add the docs and commit them
-        with tempfile.TemporaryDirectory() as tempdir:
+        with tempfile.TemporaryDirectory() as tempdir:    # pragma: no cover
             os.chdir(tempdir)
             self.clone_documentation_branch()
             if clear_before_build:
@@ -391,7 +394,7 @@ def build_documentation(documentation_formats=None):
                     logger.debug(build_log.read())
 
 
-def publish_documentation(documentation_formats=None):
+def publish_documentation(documentation_formats=None, push: bool=True):  # pragma: no cover
     """
     Publish documentation using all plugins that can generate documentation
     """
@@ -401,7 +404,7 @@ def publish_documentation(documentation_formats=None):
         if documentation_plugin.documentation_is_present:
             print(f'Publishing {documentation_plugin.name!r} documentation: ', end='', flush=True)
             try:
-                documentation_plugin.publish_documentation(clear_before_build=clear_before_build)
+                documentation_plugin.publish_documentation(clear_before_build=clear_before_build, push=push)
             except (DocBuildError, DocPublishError) as error:
                 print(colored('Failed', color='red'), flush=True)
                 if documentation_plugin.build_log_filename and os.path.exists(documentation_plugin.build_log_filename):

src/screwdrivercd/utility/environment.py

@@ -138,3 +138,21 @@ def standard_directories(command: str='') -> Dict[str, str]:
             os.makedirs(directory, exist_ok=True)
 
     return directories
+
+
+def ins_filename(filename: str) -> str:
+    """
+    Return the filename of an actual file or directory that case insensitively resolves to the filename given.  If no
+    matching files or directories exists, returns an empty string.
+    """
+    dir = os.path.dirname(filename)
+    contain_dir = dir
+    base_name = os.path.basename(filename).lower()
+    if not contain_dir:
+        contain_dir = '.'
+    if not os.path.exists(contain_dir):
+        return ''
+    for f in os.listdir(contain_dir):
+        if base_name == f.lower():
+            return os.path.join(dir, f)
+    return ''

src/screwdrivercd/validation/validate_style.py

@@ -19,6 +19,8 @@
 from termcolor import colored
 
 from ..utility import create_artifact_directory
+from ..utility.environment import ins_filename
+from ..utility.output import print_error
 from ..utility.package import PackageMetadata
 
 
@@ -57,9 +59,16 @@ def validate_with_codestyle(report_dir):
 
     # Add targets
     if src_dir not in ['', '.'] and src_dir != package_name:
-        command.append(os.path.join(src_dir, package_name.replace('.', '/')))
+        target = os.path.join(src_dir, package_name.replace('.', '/'))
     else:
-        command.append(package_name.replace('.', '/'))
+        target = package_name.replace('.', '/')
+
+    print(f'target: {target}')
+    target = ins_filename(target)
+    if not target:
+        print_error(f'ERROR: Unable to find package directory for package {package_name!r}, target directory {target!r} does not exist')
+        return 1
+    command.append(target)
 
     print('-' * 90 + '\nRunning:', ' '.join(command) + '\n' + '-' * 90, flush=True)
     rc = 0
@@ -110,7 +119,7 @@ def validate_codestyle():
         print(colored('OK: Code style validation successful', 'green'), flush=True)
 
     if rc > 0:
-        print(colored('ERROR: Type code style check failed', 'red'), file=sys.stderr, flush=True)
+        print(colored('ERROR: Code style check failed', 'red'), file=sys.stderr, flush=True)
         return rc
 
     return rc

tests/test_documentation_plugin.py

@@ -1,9 +1,11 @@
 # Copyright 2019, Oath Inc.
 # Licensed under the terms of the Apache 2.0 license.  See the LICENSE file in the project root for terms
+import copy
 import os
 import tempfile
 import unittest
 from pathlib import Path
+
 import screwdrivercd.documentation.exceptions
 import screwdrivercd.documentation.plugin
 import screwdrivercd.documentation.mkdocs.plugin
@@ -12,6 +14,30 @@
 from . import ScrewdriverTestCase
 
 
+# Simple sphinx project directory
+sphinx_project =  {
+    'doc/source/conf.py': b"""# -*- coding: utf-8 -*-
+import os
+extensions = []
+templates_path = ['_templates']
+source_suffix = '.rst'
+master_doc = 'index'
+project = 'foo'
+copyright = ''
+version = '0.0.0'
+release = version
+""",
+    'doc/source/index.rst': b"""Test file"""
+}
+
+
+mkdocs_project_config = {
+    'mkdocs.yml': b'site_name: test\nstrict: true\nnav:\n    - foo: foo.md\n',
+    'docs/index.md': b"# Test file\n",
+    'docs/foo.md': b"# Test file\n",
+}
+
+
 class PluginsTestCase(ScrewdriverTestCase):
 
     def test__documentation_plugins__present(self):
@@ -66,14 +92,22 @@ def test_documentation_base_plugin_is_present(self):
         p = screwdrivercd.documentation.plugin.DocumentationPlugin()
         self.assertFalse(p.documentation_is_present)
 
-    def test_documentation_base__log_message(self):
+    def test_documentation_base__log_message__default(self):
         p = self.plugin_class()
         p.build_log_filename = 'foo.log'
         p._log_message('foo', p.build_log_filename)
         self.assertTrue(os.path.exists('foo.log'))
         with open('foo.log') as log:
             self.assertEqual('foo\n', log.read())
 
+    def test_documentation_base__log_message__log_filename(self):
+        p = self.plugin_class()
+        p.build_log_filename = 'foo2.log'
+        p._log_message('foo', log_filename='foo.log')
+        self.assertTrue(os.path.exists('foo.log'))
+        with open('foo.log') as log:
+            self.assertEqual('foo\n', log.read())
+
     def test_documentation_base__log_message__pass_end(self):
         p = self.plugin_class()
         p._log_message('foo', log_filename='logs/foo.log', end='\r\n')
@@ -105,6 +139,7 @@ def test_documentation__base__run_command__error(self):
 
     def test__documentation__build(self):
         self._create_test_repo_contents()
+        self.write_config_files(mkdocs_project_config)
         p = self.plugin_class()
         p.build_documentation()
         self.assertTrue(os.path.exists(f'{p.log_dir}/{p.name}.build.log'))
@@ -153,22 +188,32 @@ def test__copy_contents(self):
         self.assertTrue(dsttestfile.exists())
         self.assertTrue(dsttestdotfile.exists())
 
+    def test__documentation__publish(self):
+        instance = self.plugin_class()
+        instance.publish_documentation(push=False)
+
 
 class SphinxDocumentationPluginTestCase(DocumentationPluginTestCase):
     plugin_class = screwdrivercd.documentation.sphinx.plugin.SphinxDocumentationPlugin
 
     def _create_test_repo_contents(self):
-        os.makedirs('doc/source')
+        pass
 
     def test__documentation__build(self):
         pass
 
+    def test__documentation__publish(self):
+        self.write_config_files(sphinx_project)
+        super().test__documentation__publish()
+
 
 class MkdocsDocumentationPluginTestCase(DocumentationPluginTestCase):
     plugin_class = screwdrivercd.documentation.mkdocs.plugin.MkDocsDocumentationPlugin
 
     def _create_test_repo_contents(self):
-        os.makedirs('docs')
-
-    def test__documentation__build(self):
         pass
+
+    def test__documentation__publish(self):
+        self._init_test_repo()
+        self.write_config_files(mkdocs_project_config)
+        super().test__documentation__publish()

tests/test_utility.py

@@ -7,11 +7,12 @@
 import time
 
 from datetime import timedelta
+from pathlib import Path
 from pprint import pprint
 
 from screwdrivercd.screwdriver.environment import update_job_status
 from screwdrivercd.utility.contextmanagers import InTemporaryDirectory, Timeout, revert_file, working_dir
-from screwdrivercd.utility.environment import env_bool, env_int, flush_terminals, interpreter_bin_command, standard_directories
+from screwdrivercd.utility.environment import env_bool, env_int, flush_terminals, ins_filename, interpreter_bin_command, standard_directories
 from screwdrivercd.utility.exceptions import TimeoutError
 from screwdrivercd.utility.output import header, print_error, status_message
 from screwdrivercd.utility.package import run_setup_command, setup_query, PackageMetadata
@@ -169,6 +170,26 @@ def test__create_artifact_directory_passed(self):
         create_artifact_directory(f'{self.tempdir.name}/test_artifacts')
         self.assertTrue(os.path.exists(f'{self.tempdir.name}/test_artifacts'))
 
+    def test__ins_filename__nodir(self):
+        Path('foo').mkdir()
+        result = ins_filename('foo')
+        self.assertEqual(result, 'foo')
+
+    def test__ins_filename__nodir__differentcase(self):
+        Path('Goo').mkdir()
+        result = ins_filename('goo')
+        self.assertEqual(result, 'Goo')
+
+    def test__ins_filename__fullpath(self):
+        Path('foo').mkdir()
+        fullpath = os.path.join(self.tempdir.name, 'foo')
+        result = ins_filename(fullpath)
+        self.assertEqual(result, fullpath)
+
+    def test__ins_filename__dirnoexist(self):
+        result = ins_filename('foo')
+        self.assertEqual(result, '')
+
     def test__run__run_log_output__success(self):
         with InTemporaryDirectory():
             result = run_and_log_output(['echo', 'hello'], 'echo.log')

tests/test_validators.py

@@ -87,6 +87,17 @@ def test_dummy(self):
     mypyvalidator
 """
 
+# src dir has different case than package name
+working_config_case_different = copy.deepcopy(working_config)
+working_config_case_different['src/MyPyvalidator/__init__.py'] = working_config['src/mypyvalidator/__init__.py']
+del working_config_case_different['src/mypyvalidator/__init__.py']
+working_config_case_different['setup.cfg'].replace(b'\n    mypyvalidator', b'\n    MyPyvalidator')
+
+# No source dir different case
+working_config_case_different_nosrc = copy.deepcopy(working_config_case_different)
+working_config_case_different_nosrc['MyPyvalidator/__init__.py'] = working_config_case_different_nosrc['src/MyPyvalidator/__init__.py']
+del working_config_case_different_nosrc['src/MyPyvalidator/__init__.py']
+
 # Invalid package tree
 invalid_type_config = copy.deepcopy(working_config)
 invalid_type_config['src/mypyvalidator/__init__.py'] = b"""a: int='1'"""
@@ -180,6 +191,26 @@ def test__style__pass__nosrc(self):
         result = validate_codestyle()
         self.assertEqual(result, 0)
 
+    def test_style__pass__srcdir_wrong_case(self):
+        os.environ['PACKAGE_DIR'] = 'src'
+        self.write_config_files(working_config_case_different)
+        os.system('ls -lR')
+        result = validate_codestyle()
+        self.assertEqual(result, 0)
+
+    def test_style__pass__srcdir_wrong_case__missing(self):
+        os.environ['PACKAGE_DIR'] = 'srcz'
+        self.write_config_files(working_config_case_different)
+        os.system('ls -lR')
+        result = validate_codestyle()
+        self.assertNotEquals(result, 0)
+
+    def test_style__pass__nosrcdir_wrong_case(self):
+        self.write_config_files(working_config_case_different_nosrc)
+        os.system('ls -lR')
+        result = validate_codestyle()
+        self.assertEqual(result, 0)
+
 
 class TypeValidatorTestcase(ScrewdriverTestCase):
     validator_name = 'type_validation'