Deprecate python2.7 (remove python2.7 testing). (#430)

Author: DailyDreaming

.gitlab-ci.yml

@@ -48,10 +48,6 @@ stages:
     - (Get-Content .\hca\default_config.json).replace('dss.data.humancellatlas.org',"dss.${DSS_TEST_STAGE}.data.humancellatlas.org")| Set-Content  .\hca\default_config.json
   script:
     - make test-win
-test-2.7:
-  extends: .linux-test-base
-  variables:
-    PYTHON_VERSION: "2.7"
 
 test-3.5:
   extends: .linux-test-base

.travis.yml

@@ -3,7 +3,6 @@ sudo: required
 dist: xenial
 cache: pip
 python:
-- 2.7
 - 3.5
 - 3.6
 - 3.7

README.rst

@@ -6,6 +6,8 @@ Data Coordination Platform (DCP) of the Human Cell Atlas (HCA).
 Currently the `hca` package supports interaction with the `Upload Service <https://github.com/HumanCellAtlas/upload-service>`_ and `Data Storage Service (DSS) <https://github.com/HumanCellAtlas/data-store>`_ for services such as uploading, downloading,
 and querying data.
 
+The HCA CLI is compatible with python versions 3.5+ (we are no longer compatible with python2.7, and our last compatible python2.7 version was `hca==6.4.0`).
+
 Installation
 ------------
 :code:`pip install hca`.

hca/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 from .config import HCAConfig, get_config, logger
 from . import dss, upload, query
 

hca/cli.py

@@ -5,8 +5,6 @@
 For general help, run ``{prog} help``.
 For help with individual commands, run ``{prog} <command> --help``.
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import os
 import sys
 import argparse
@@ -18,19 +16,13 @@
 import argcomplete
 from io import open
 from botocore.exceptions import NoRegionError
-
-try:
-    import xmlrpclib
-except ImportError:
-    import xmlrpc.client as xmlrpclib
+import xmlrpc.client as xmlrpclib
 
 from .version import __version__
 from .dss import cli as dss_cli
 from .upload import cli as upload_cli
 from .query import cli as query_cli
-from .util.compat import USING_PYTHON2
-from . import clear_hca_cache
-from . import logger, get_config
+from . import logger, get_config, clear_hca_cache
 
 
 class HCAArgumentParser(argparse.ArgumentParser):
@@ -47,8 +39,7 @@ def add_parser_func(self, func, **kwargs):
         subparser.set_defaults(**get_config().get(command, {}))
         if subparser.description is None:
             subparser.description = kwargs.get("help", func.__doc__)
-        if sys.version_info < (2, 7, 9):  # See https://bugs.python.org/issue9351
-            self._defaults.pop("entry_point", None)
+        self._defaults.pop("entry_point", None)
         return subparser
 
     def print_help(self, file=None):
@@ -161,7 +152,6 @@ def main(args=None):
         raise result
     elif result is not None:
         if isinstance(result, bytes):
-            out_stream = sys.stdout if USING_PYTHON2 else sys.stdout.buffer
-            out_stream.write(result)
+            sys.stdout.buffer.write(result)
         elif not isinstance(result, upload_cli.UploadCLICommand):
             print(json.dumps(result, indent=2, default=lambda x: str(x)))

hca/dss/__init__.py

@@ -2,14 +2,10 @@
 Data Storage System
 *******************
 """
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import errno
 import functools
 import json
-import threading
-from collections import defaultdict, namedtuple, OrderedDict
+from collections import defaultdict, namedtuple
 import csv
 import concurrent.futures
 from datetime import datetime
@@ -27,8 +23,7 @@
 from requests.exceptions import ChunkedEncodingError, ConnectionError, ReadTimeout
 
 from hca.dss.util import iter_paths, object_name_builder, hardlink
-from hca.util import USING_PYTHON2
-from hca.util.compat import glob_escape
+from glob import escape as glob_escape
 from ..util import SwaggerClient, DEFAULT_THREAD_COUNT
 from ..util.exceptions import SwaggerAPIException
 from .. import logger
@@ -400,7 +395,7 @@ def download_collection(self, uuid, replica, version=None, download_dir=''):
                                              'file_uuid',
                                              'file_version',
                                              'file_size'),
-                                 delimiter=str('\t'),  # cast for py2.7 compat
+                                 delimiter='\t',
                                  quoting=csv.QUOTE_NONE)
             tsv.writeheader()
             tsv.writerows(collection)
@@ -472,7 +467,6 @@ class DownloadContext(object):
     # This variable is the configuration for download_manifest_v2. It specifies the length of the names of nested
     # directories for downloaded files.
     DIRECTORY_NAME_LENGTHS = [2, 4]
-    MANIFEST_DELIMITER = b'\t' if USING_PYTHON2 else '\t'
 
     def __init__(self, download_dir, dss_client, replica, num_retries, min_delay_seconds):
         self.runner = TaskRunner()
@@ -535,7 +529,7 @@ def _download_bundle_manifest(self, manifest_bytes, bundle_dir, dss_file):
 
     def _get_full_bundle_manifest(self, bundle_uuid, version):
         """
-        Takes care of paging through the bundle and checks for name collisions
+        Takes care of paging through the bundle and checks for name collisions.
         """
         pages = self.dss_client.get_bundle.paginate(uuid=bundle_uuid,
                                                     version=version if version else None,
@@ -734,8 +728,7 @@ def _download_manifest_tasks(self, no_metadata, no_data):
         with open(self.manifest) as f:
             bundles = defaultdict(set)
             # unicode_literals is on so all strings are unicode. CSV wants a str so we need to jump through a hoop.
-            delimiter = self.MANIFEST_DELIMITER
-            reader = csv.DictReader(f, delimiter=delimiter, quoting=csv.QUOTE_NONE)
+            reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
             for row in reader:
                 bundles[(row['bundle_uuid'], row['bundle_version'])].add(row['file_name'])
         for (bundle_uuid, bundle_version), data_files in bundles.items():
@@ -761,8 +754,7 @@ def _write_output_manifest(self):
         if 'file_path' not in fieldnames:
             fieldnames.append('file_path')
         with atomic_write(output, overwrite=True) as f:
-            delimiter = b'\t' if USING_PYTHON2 else '\t'
-            writer = csv.DictWriter(f, fieldnames, delimiter=delimiter, quoting=csv.QUOTE_NONE)
+            writer = csv.DictWriter(f, fieldnames, delimiter='\t', quoting=csv.QUOTE_NONE)
             writer.writeheader()
             for row in source_manifest:
                 row['file_path'] = self._file_path(row['file_sha256'], self.download_dir)
@@ -775,6 +767,5 @@ def _write_output_manifest(self):
     def _parse_manifest(cls, manifest):
         with open(manifest) as f:
             # unicode_literals is on so all strings are unicode. CSV wants a str so we need to jump through a hoop.
-            delimiter = cls.MANIFEST_DELIMITER
-            reader = csv.DictReader(f, delimiter=delimiter, quoting=csv.QUOTE_NONE)
+            reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
             return reader.fieldnames, list(reader)

hca/dss/cli.py

@@ -1,7 +1,3 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import sys
-
 from . import DSSClient
 
 def add_commands(subparsers, help_menu=False):
@@ -10,8 +6,7 @@ def add_commands(subparsers, help_menu=False):
     def help(args):
         dss_parser.print_help()
 
-    if sys.version_info >= (2, 7, 9):  # See https://bugs.python.org/issue9351
-        dss_parser.set_defaults(entry_point=help)
+    dss_parser.set_defaults(entry_point=help)
     dss_subparsers = dss_parser.add_subparsers()
     dss_cli_client = DSSClient()
     dss_cli_client.build_argparse_subparsers(dss_subparsers, help_menu=help_menu)

hca/dss/infra.py

@@ -1,8 +1,5 @@
 #!/usr/bin/env python
 # coding: utf-8
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 import logging
 
 

hca/dss/util/__init__.py

@@ -1,9 +1,5 @@
-import errno
 import os
-
-import platform
-import sys
-
+from builtins import FileExistsError
 from ...util.compat import scandir
 
 
@@ -44,26 +40,14 @@ def hardlink(source, link_name):
     The code for Windows support is adapted from:
     https://github.com/sunshowers/ntfs/blob/master/ntfsutils/hardlink.py
     """
-    if sys.version_info < (3,) and platform.system() == 'Windows':  # pragma: no cover
-        import ctypes
-        create_hard_link = ctypes.windll.kernel32.CreateHardLinkW
-        create_hard_link.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p]
-        create_hard_link.restype = ctypes.wintypes.BOOL
-        res = create_hard_link(link_name, source, None)
-        if res == 0:
-            raise ctypes.WinError()
-    else:
-        try:
-            os.link(source, link_name)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-            else:
-                # It's possible that the user created a different file with the same name as the
-                # one we're trying to download. Thus we need to check the if the inode is different
-                # and raise an error in this case.
-                source_stat = os.stat(source)
-                dest_stat = os.stat(link_name)
-                # Check device first because different drives can have the same inode number
-                if source_stat.st_dev != dest_stat.st_dev or source_stat.st_ino != dest_stat.st_ino:
-                    raise
+    try:
+        os.link(source, link_name)
+    except FileExistsError:
+        # It's possible that the user created a different file with the same name as the
+        # one we're trying to download. Thus we need to check the if the inode is different
+        # and raise an error in this case.
+        source_stat = os.stat(source)
+        dest_stat = os.stat(link_name)
+        # Check device first because different drives can have the same inode number
+        if source_stat.st_dev != dest_stat.st_dev or source_stat.st_ino != dest_stat.st_ino:
+            raise

hca/query/__init__.py

@@ -2,9 +2,6 @@
 DCP Query Service
 *****************
 """
-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
 from ..util import SwaggerClient