Merge pull request #27 from cloudify-incubator/python3

Python3

Author: EarthmanT

.circleci/config.yml

@@ -11,7 +11,34 @@ checkout:
 
 jobs:
 
-  unittests:
+  py3_compat:
+    docker:
+      - image: circleci/python:2.7
+    steps:
+      - checkout
+      - run:
+          name: install futurize
+          command: pip install future --user
+      - run:
+          name: find python3-incompatible code
+          command: |
+            FUTURIZE="futurize ."
+            while read line; do
+              [[ "$line" =~ ^#.* ]] && continue
+              FUTURIZE="${FUTURIZE} ${line}"
+            done<.circleci/py3fixers
+            echo "Running: $FUTURIZE"
+            $FUTURIZE>futurize_diffs
+      - run:
+          name: check that there is no python3-incompatible code
+          command: |
+            if [[ -s futurize_diffs ]]; then
+              echo "Python-3-incompatible code found"
+              cat futurize_diffs
+              exit 1
+            fi
+
+  unittests_py27:
     docker:
       - image: circleci/python:2.7.15-stretch
     steps:
@@ -38,54 +65,138 @@ jobs:
           name: install tox
           command: pip install --user tox
       - run: /home/circleci/.local/bin/tox -e flake8
-      - run: /home/circleci/.local/bin/tox -e py27
+      - run: /home/circleci/.local/bin/tox -e nosetests
 
-  wagon:
+  unittests_py36:
     docker:
-      - image: amd64/centos:centos7.5.1804
+      - image: circleci/python:3.6-stretch
     steps:
       - checkout
       - run:
-          name: Install dependencies
-          command: yum -y install python-devel gcc openssl git libxslt-devel libxml2-devel openldap-devel libffi-devel openssl-devel libvirt-devel libvirt-python
+          name: update links
+          command: sudo apt-get update -y
+      - run:
+          name: install python/pip/libvirt packages
+          command: sudo apt-get install -yq python-libvirt libvirt-dev python-dev python-pip
       - run:
           name: Download pip
           command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
       - run:
           name: Install pip
-          command: python get-pip.py
-      - run:
-          name: Upgrade pip
-          command: pip install --upgrade pip==9.0.1
+          command: sudo python get-pip.py
       - run:
           name: Install virtualenv
-          command: pip install virtualenv==16.7.9
+          command: pip install --user virtualenv
       - run:
           name: Init virtualenv
           command: virtualenv env
       - run:
-          name: Install wagon
-          command: pip install wagon==0.3.2
+          name: install tox
+          command: pip install --user tox
+      - run: /home/circleci/.local/bin/tox -e flake8
+      - run: /home/circleci/.local/bin/tox -e nosetests
+
+  wagon:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run:
+          name: Create Workspace Build directory.
+          command: mkdir -p workspace/build
       - run:
-          name: many_linux
-          command: echo "manylinux1_compatible = False" > "env/bin/_manylinux.py"
+          name: Build py27py36 Wagon
+          command: |
+              git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
+              docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7_py2py3
+              docker run -v ~/project/:/packaging cloudify-centos-7-wagon-builder
       - run:
-          name: make workspace
+          name: copy wagon to workspace
+          command: cp *.wgn workspace/build/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - build/*
+
+  rhel_wagon:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run:
+          name: Create Workspace Build directory.
           command: mkdir -p workspace/build
       - run:
-          name: Create wagon
-          command: source env/bin/activate && wagon create -s . -v -o workspace/build -f -a '--no-cache-dir -c constraints.txt'
+          name: Build RHEL py27py36 Wagon
+          command: |
+              git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git
+              docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7_py2py3 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD"
+              docker run -v ~/project/:/packaging cloudify-redhat-7-wagon-builder
+      - run:
+          name: copy wagon to workspace
+          command: cp *.wgn workspace/build/
       - persist_to_workspace:
           root: workspace
           paths:
             - build/*
 
+  release:
+    docker:
+      - image: circleci/python:2.7
+    steps:
+      - checkout
+      - run:
+          name: "Pull Submodules"
+          command: |
+            git submodule init
+            git submodule update --remote --recursive
+      - run:
+          name: Download pip
+          command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
+      - run:
+          name: Install pip
+          command: sudo python get-pip.py
+      - run:
+          name: Install virtualenv
+          command: pip install --user virtualenv
+      - run:
+          name: Init virtualenv
+          command: virtualenv env
+      - run:
+          name: install tox
+          command: pip install --user pygithub pyyaml==3.10
+      - run:
+          name: upgrade setuptools
+          command: pip install --upgrade setuptools
+      - run:
+          name: install local project
+          command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip
+      - attach_workspace:
+          at: workspace
+      - run: python .circleci/package_release.py
+
+
 workflows:
   version: 2
   tests:
     jobs:
-      - unittests
+      - py3_compat
+      - unittests_py27
+      - unittests_py36
       - wagon:
           filters:
             branches:
               only: /([0-9\.]*\-build|master|dev)/
+      - rhel_wagon:
+          filters:
+            branches:
+              only: /([0-9\.]*\-build|master|dev)/
+      - release:
+          filters:
+            branches:
+              only: /master/
+          requires:
+            - unittests_py27
+            - unittests_py36
+            - wagon
+            - rhel_wagon

.circleci/py3fixers

@@ -0,0 +1,23 @@
+--stage1
+-f lib2to3.fixes.fix_getcwdu
+-f lib2to3.fixes.fix_long
+-f lib2to3.fixes.fix_nonzero
+-f lib2to3.fixes.fix_input
+-f lib2to3.fixes.fix_raw_input
+-f lib2to3.fixes.fix_itertools
+-f lib2to3.fixes.fix_itertools_imports
+-f lib2to3.fixes.fix_exec
+-f lib2to3.fixes.fix_operator
+-f libfuturize.fixes.fix_execfile
+-f libpasteurize.fixes.fix_newstyle
+-f lib2to3.fixes.fix_filter
+# fix_dict is not idempotent
+# -f lib2to3.fixes.fix_dict
+-f lib2to3.fixes.fix_map
+-f lib2to3.fixes.fix_zip
+-f lib2to3.fixes.fix_xrange
+-f lib2to3.fixes.fix_basestring
+-f libfuturize.fixes.fix_cmp
+-f libfuturize.fixes.fix_division_safe
+-f lib2to3.fixes.fix_metaclass
+-f libfuturize.fixes.fix_unicode_keep_u

CHANGELOG.txt

@@ -43,3 +43,7 @@ releases:
 
   v0.9.0:
    * Support download image contwent from external resource
+
+  v0.9.1:
+  * Apply futurize fixes for python3
+  * Port source code to python3

README.md

@@ -11,6 +11,7 @@ Release history: [CHANGELOG.txt](CHANGELOG.txt)
 
 * Python versions:
     * 2.7.x
+    * 3.6.x
 * Packages versions:
     * libvirt-python >= 3.7.0
     * libvirt >= 1.3.1

cloudify_libvirt/common.py

@@ -17,8 +17,8 @@
 
 from cloudify import ctx
 from cloudify import exceptions as cfy_exc
-
 from cloudify_common_sdk import filters
+from cloudify_common_sdk._compat import text_type
 
 
 def get_libvirt_params(**kwargs):
@@ -40,7 +40,7 @@ def get_libvirt_params(**kwargs):
     if not template_params.get("name"):
         template_params["name"] = ctx.instance.id
     if not template_params.get("instance_uuid"):
-        template_params["instance_uuid"] = str(uuid.uuid4())
+        template_params["instance_uuid"] = text_type(uuid.uuid4())
 
     # update 'resource_id', 'use_external_resource' from kwargs
     for field in ['resource_id', 'use_external_resource']:

cloudify_libvirt/domain_tasks.py

@@ -16,8 +16,9 @@
 import time
 
 from cloudify import ctx
-from cloudify.decorators import operation
 from cloudify import exceptions as cfy_exc
+from cloudify.decorators import operation
+from cloudify_common_sdk._compat import text_type
 import cloudify_libvirt.common as common
 
 
@@ -270,9 +271,9 @@ def start(**kwargs):
                 'Failed to find the domain: {}'.format(repr(e))
             )
 
-        for i in xrange(10):
+        for i in range(10):
             state, _ = dom.state()
-            ctx.logger.info("Tring to start vm {}/10".format(i))
+            ctx.logger.info("Trying to start vm {}/10".format(i))
             if wait_for_ip:
                 ctx.logger.info("Waiting for ip.")
             if state == libvirt.VIR_DOMAIN_RUNNING:
@@ -333,12 +334,12 @@ def stop(**kwargs):
         ctx.instance.runtime_properties['ip'] = None
 
         state, _ = dom.state()
-        for i in xrange(10):
+        for i in range(10):
             if state != libvirt.VIR_DOMAIN_RUNNING:
                 ctx.logger.info("Looks as not run.")
                 return
 
-            ctx.logger.info("Tring to stop vm {}/10".format(i))
+            ctx.logger.info("Trying to stop vm {}/10".format(i))
             if dom.shutdown() < 0:
                 raise cfy_exc.NonRecoverableError(
                     'Can not shutdown guest domain.'
@@ -375,12 +376,12 @@ def resume(**kwargs):
             )
 
         state, _ = dom.state()
-        for i in xrange(10):
+        for i in range(10):
             if state == libvirt.VIR_DOMAIN_RUNNING:
                 ctx.logger.info("Looks as running.")
                 return
 
-            ctx.logger.info("Tring to resume vm {}/10".format(i))
+            ctx.logger.info("Trying to resume vm {}/10".format(i))
             if dom.resume() < 0:
                 raise cfy_exc.NonRecoverableError(
                     'Can not suspend guest domain.'
@@ -417,12 +418,12 @@ def suspend(**kwargs):
             )
 
         state, _ = dom.state()
-        for i in xrange(10):
+        for i in range(10):
             if state != libvirt.VIR_DOMAIN_RUNNING:
                 ctx.logger.info("Looks as not run.")
                 return
 
-            ctx.logger.info("Tring to suspend vm {}/10".format(i))
+            ctx.logger.info("Trying to suspend vm {}/10".format(i))
             if dom.suspend() < 0:
                 raise cfy_exc.NonRecoverableError(
                     'Can not suspend guest domain.'
@@ -437,7 +438,7 @@ def _cleanup_snapshots(ctx, dom):
     snapshots = dom.listAllSnapshots()
     snapshots_count = len(snapshots)
 
-    for _ in xrange(snapshots_count):
+    for _ in range(snapshots_count):
         for snapshot in snapshots:
             # we can delete only snapshot without child
             if not snapshot.numChildren():
@@ -476,7 +477,7 @@ def _delete_force(dom):
                 'Can not undefine guest domain with NVRAM.'
             )
     except AttributeError as e:
-        ctx.logger.info("Non critical error: {}".format(str(e)))
+        ctx.logger.info("Non critical error: {}".format(text_type(e)))
         if dom.undefine() < 0:
             raise cfy_exc.RecoverableError(
                 'Can not undefine guest domain.'
@@ -795,7 +796,7 @@ def perfomance(**kwargs):
         # and after sleep for 5 seconds.
         ctx.logger.debug("Used: {} seconds.".format(before_usage))
         time.sleep(5)
-        statistics['cpu'] = 100 * (_current_use(dom) - before_usage) / 5
+        statistics['cpu'] = 100 * (_current_use(dom) - before_usage) // 5
 
         memory = dom.memoryStats()
         statistics['memory'] = memory.get('actual', 0) / 1024.0

cloudify_libvirt/network_tasks.py

@@ -219,7 +219,7 @@ def link(**kwargs):
             )
 
         MAX_RETRY = 10
-        for i in xrange(MAX_RETRY):
+        for i in range(MAX_RETRY):
             ctx.logger.info("{}: Tring to get vm ip: {}/{}"
                             .format(vm_id, i, MAX_RETRY))
             for lease in network.DHCPLeases():