Use packages for style validation (#99)

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Update setup.py

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

* Make validate_style actually use the directories for the packages from the package metadata.

Co-authored-by: Dwight Hubbard <dhubbard@verizonmedia.com>

Author: dwighthubbard

changelog.d/94.feature.md

@@ -0,0 +1 @@
+True otherwise

screwdriver.yaml

@@ -5,17 +5,20 @@ version: 4
 shared:
     environment:
         CHANGELOG_FILENAME: docs/changelog.md
-        DOCUMENTATION_DEBUG: False
+        DOCUMENTATION_DEBUG: True
         DOCUMENTATION_FORMATS: mkdocs
-        PACKAGE_DIR: src
-        PACKAGE_DIRECTORY: src
+        # PACKAGE_DIR: src
+        # PACKAGE_DIRECTORY: src
     steps:
-        - presetup_deploy_keys: $BASE_PYTHON -m pip install -U .[documentation]
         - tag_release: $BASE_PYTHON -m screwdrivercd.repo.release
 
 jobs:
     validate_codestyle:
         template: python/validate_codestyle
+        environment:
+            CODESTYLE_ARGS: -v
+        steps:
+            - postinstall_dependencies: $BASE_PYTHON -m pip install -U .
         requires: [~commit, ~pr]
 
     validate_dependencies:
@@ -45,12 +48,6 @@ jobs:
     #    template: python/validate_type
     #     requires: [~commit, ~pr]
 
-    validate_documentation:
-        template: python/documentation
-        environment:
-            DOCUMENTATION_PUBLISH: False
-        requires: [~pr]
-
     version:
         template: python/generate_version
         requires: [
@@ -79,4 +76,8 @@ jobs:
 
     publish_documentation:
         template: python/documentation
-        requires: [publish_pypi]
+        steps:
+            - postinstall_dependencies: |
+                $BASE_PYTHON -m pip install -U .
+                printenv | grep SD | sort
+        requires: [~pr, ~publish_pypi]

setup.cfg

@@ -86,36 +86,29 @@ screwdrivercd.documentation.plugin =
 
 [options.extras_require]
 documentation = 
-	dhubbard-sphinx-markdown-tables
+	# dhubbard-sphinx-markdown-tables
 	# markdown<3.2    # This is only needed for the mkdocs-material package to install
 	# pymdown-extensions<6.3  # This is needed to allow markdown < 3.2 to install
 	markdown
 	pymdown-extensions
-	guzzle_sphinx_theme
 	markdown-include
 	mkdocs-bootstrap4
 	mkdocs-cinder
 	mkdocs-material
-	nbsphinx
 	pygments
-
 	recommonmark
-	sphinx_rtd_theme
-doc_build = 
-	dhubbard-sphinx-markdown-tables
+doc_build =
+	# dhubbard-sphinx-markdown-tables
 	# markdown<3.2    # This is only needed for the mkdocs-material package to install
 	# pymdown-extensions<6.3  # This is needed to allow markdown < 3.2 to install
 	markdown
 	pymdown-extensions
-	guzzle_sphinx_theme
 	markdown-include
 	mkdocs-bootstrap4
 	mkdocs-material
-	nbsphinx
 	pygments
 	recommonmark
-	sphinx_rtd_theme
-test = 
+test =
 	pytest
 	pytest-cov
 	pypirun

setup.py

@@ -5,22 +5,5 @@
 Package setup file for python module screwdrivercd
 """
 import setuptools
-import sys
-
-
-def setuptools_version_supported():
-    major, minor, patch = setuptools.__version__.split('.')
-    if int(major) > 38:
-        return True
-    return False
 
-
-if __name__ == '__main__':
-    # Check for a working version of setuptools here because earlier versions did not
-    # support python_requires.
-    if not setuptools_version_supported():
-        print('Setuptools version 38.0.0 or higher is needed to install this package')
-        sys.exit(1)
-
-    # We're being run from the command line so call setup with our arguments
-    setuptools.setup()
+setuptools.setup()

src/screwdrivercd/documentation/cli.py

@@ -9,11 +9,12 @@
 logging_basicConfig(check_prefix='DOCUMENTATION')
 import logging
 import os
-import sys
+
+from termcolor import colored
 
 from .exceptions import DocBuildError, DocPublishError
 from .plugin import build_documentation, publish_documentation
-from ..utility import env_bool
+from ..utility.environment import env_bool, is_pull_request
 
 
 logger_name = __name__
@@ -36,15 +37,24 @@ def main():  # pragma: no cover
     if documentation_formats:
         documentation_formats = [_.strip() for _ in documentation_formats.split(',')]
 
-    if env_bool('DOCUMENTATION_PUBLISH', True):  # pragma: no cover
+    documentation_publish_default = False if is_pull_request() else True
+
+    rc = 0
+    if env_bool('DOCUMENTATION_PUBLISH', documentation_publish_default):  # pragma: no cover
+        operation = 'Published'
         try:
             publish_documentation(documentation_formats=documentation_formats)
         except DocPublishError:
-            return 1
+            rc = 1
+        if is_pull_request():
+            print(colored('Warning: Published documentation from a pull request', 'yellow'))
     else:
+        operation = 'Generated'
         try:
             build_documentation(documentation_formats=documentation_formats)
         except DocBuildError:
-            return 1
-    update_job_status(status='SUCCESS', message=f'Generated {", ".join(documentation_formats)} documentation')
-    return 0
+            rc = 1
+    if is_pull_request():
+        status = 'SUCCESS' if rc == 0 else 'FAILURE'
+        update_job_status(status=status, message=f'{operation} {", ".join(documentation_formats)} documentation')
+    return rc

src/screwdrivercd/screwdriver/environment.py

@@ -7,7 +7,7 @@
 import logging
 import os
 import shutil
-from ..utility import env_bool
+from ..utility.environment import env_bool, is_pull_request
 from .metadata import Metadata
 
 

src/screwdrivercd/utility/environment.py

@@ -77,7 +77,7 @@ def flush_terminals():
     sys.stderr.flush()
 
 
-def interpreter_bin_command(command: str = 'python', fallback_path: bool=True) -> str:
+def interpreter_bin_command(command: str='', fallback_path: bool=True) -> str:
     """
     Return the full path to a command in the current interpreter's bin directory.
 
@@ -94,6 +94,8 @@ def interpreter_bin_command(command: str = 'python', fallback_path: bool=True) -
         Full path to the command.  If the command is not present returns command if fallback_path is True or an empty
         string otherwise.
     """
+    if not command:
+        command = os.path.basename(sys.executable)
     bin_dir = os.path.dirname(sys.executable)
     new_command = os.path.join(bin_dir, command)
     if os.path.exists(new_command):
@@ -145,14 +147,32 @@ def ins_filename(filename: str) -> str:
     Return the filename of an actual file or directory that case insensitively resolves to the filename given.  If no
     matching files or directories exists, returns an empty string.
     """
-    dir = os.path.dirname(filename)
-    contain_dir = dir
+    contain_dir = os.path.dirname(filename)
     base_name = os.path.basename(filename).lower()
     if not contain_dir:
         contain_dir = '.'
     if not os.path.exists(contain_dir):
         return ''
     for f in os.listdir(contain_dir):
         if base_name == f.lower():
-            return os.path.join(dir, f)
+            if os.path.dirname(filename):
+                return os.path.join(os.path.dirname(filename), f)
+            else:
+                return f
     return ''
+
+
+def is_pull_request() -> bool:
+    """
+    Return True if the SD_PULL_REQUEST environment variable has a PULL Request value, False otherwise
+    """
+    pr_num_str = os.environ.get('SD_PULL_REQUEST', '')
+    if not pr_num_str:
+        return False
+
+    try:
+        pr_num = int(pr_num_str)
+    except ValueError:
+        return False
+
+    return True

src/screwdrivercd/utility/package.py

@@ -174,3 +174,36 @@ def get_package_data(self, path=None):
                         self.options[option_item] = True
                     else:  # pragma: no cover
                         self.options[option_item] = False
+
+
+def package_srcdir() -> str:
+    # Package directory is being specifically set using env variables
+    env_package_dir = os.environ.get('PACKAGE_DIR', '')
+    if not env_package_dir:
+        env_package_dir = os.environ.get('PACKAGE_DIRECTORY', '')
+
+    if env_package_dir:
+        return env_package_dir
+
+    package_metadata = PackageMetadata()
+    package_dir = package_metadata.options.get('package_dir', '').strip().lstrip('=')
+
+    # Package directory specified in the package
+    if package_dir:
+        return package_dir
+
+    package_name = package_metadata.metadata['name']
+
+    # Fallback to looking for the package in a directory matching the package name
+    if os.path.exists(package_name):
+        return package_name
+
+    for entry in os.listdir('.'):
+        if entry.lower() == package_name.lower():
+            return entry
+
+    for entry in os.listdir('.'):
+        if entry.lower().replace('-', '_') == package_name.lower().replace('-', '_'):
+            return entry
+
+    return ''

src/screwdrivercd/validation/validate_style.py

@@ -21,7 +21,7 @@
 from ..utility import create_artifact_directory
 from ..utility.environment import ins_filename
 from ..utility.output import print_error
-from ..utility.package import PackageMetadata
+from ..utility.package import PackageMetadata, package_srcdir
 
 
 logger_name = 'validate_style' if __name__ == '__main__' else __name__
@@ -33,11 +33,7 @@ def validate_with_codestyle(report_dir):
 
     package_metadata = PackageMetadata()
     package_name = package_metadata.metadata['name']
-    package_dir = package_metadata.options.get('package_dir', '').strip().lstrip('=')
-
-    src_dir = os.environ.get('PACKAGE_DIR', package_dir)
-    if not src_dir:
-        src_dir = os.environ.get('PACKAGE_DIRECTORY', '')
+    src_dir = package_srcdir()
 
     parent_interpreter = interpreter_parent(sys.executable)
     interpreter = os.environ.get('BASE_PYTHON', parent_interpreter)
@@ -50,6 +46,8 @@ def validate_with_codestyle(report_dir):
         if not os.path.exists(pycodestyle_command):
             bin_dir = os.path.dirname(parent_interpreter)
             pycodestyle_command = os.path.join(bin_dir, 'pycodestyle')
+            if not os.path.exists(pycodestyle_command):
+                pycodestyle_command = 'pycodestyle'
 
     # Generate the command line from the environment settings
     command = [pycodestyle_command]
@@ -59,25 +57,30 @@ def validate_with_codestyle(report_dir):
     if extra_args:
         command += extra_args.split()
 
-    # Add targets
-    target = ''
-    for package in package_metadata.packages:
-        package_path = os.path.join(src_dir, package)
-        if os.path.exists(package_path):
-            target = package_path
-            break
-
-    if not target:
-        if src_dir not in ['', '.'] and src_dir != package_name:
-            target = os.path.join(src_dir, package_name.replace('.', '/'))
-        else:
-            target = package_name.replace('.', '/')
-
-    print(f'target: {target}')
-    target = ins_filename(target)
-    if not target:
-        print_error(f'ERROR: Unable to find package directory for package {package_name!r}, target directory {target!r} does not exist')
-        return 1
+    if src_dir and src_dir not in ['.']:
+        target = src_dir
+    else:
+        # Add targets
+        target = ''
+        src_dir = '.'
+        if hasattr(package_metadata, 'packages'):
+            for package in package_metadata.packages:
+                package_path = os.path.join(src_dir, package)
+                if os.path.exists(package_path):
+                    target = package_path
+                    break
+
+        if not target:
+            if src_dir not in ['', '.'] and src_dir != package_name:
+                target = os.path.join(src_dir, package_name.replace('.', '/'))
+            else:
+                target = package_name.replace('.', '/')
+
+        print(f'target: {target}')
+        target = ins_filename(target)
+        if not target:
+            print_error(f'ERROR: Unable to find package directory for package {package_name!r}, target directory {target!r} does not exist')
+            return 1
     command.append(target)
 
     print('-' * 90 + '\nRunning:', ' '.join(command) + '\n' + '-' * 90, flush=True)

src/screwdrivercd/validation/validate_type.py

@@ -21,7 +21,8 @@
 
 from termcolor import colored
 from ..utility import create_artifact_directory, env_bool
-from ..utility.package import PackageMetadata
+from ..utility.package import PackageMetadata, package_srcdir
+
 
 
 logger_name = 'validate_type' if __name__ == '__main__' else __name__