#533 - In progress of switch http lib to requests lib

 * In progress of updating "uriopen()" to requests.get()

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

src/attributecode/api.py

@@ -15,11 +15,10 @@
 # ============================================================================
 
 import json
+from requests import get
 
 from urllib.parse import quote
 from urllib.parse import urlencode
-from urllib.request import Request
-from urllib.request import urlopen
 from urllib.error import HTTPError
 
 from attributecode import ERROR
@@ -55,9 +54,11 @@ def request_license_data(api_url, api_key, license_key):
     license_data = {}
     errors = []
     try:
-        request = Request(quoted_url, headers=headers)
-        response = urlopen(request)
-        response_content = response.read().decode('utf-8')
+        # request = Request(quoted_url, headers=headers)
+        # response = urlopen(request)
+        # response_content = response.read().decode('utf-8')
+        response = get(quoted_url, headers=headers)
+        response_content = response.text
         # FIXME: this should be an ordered dict
         license_data = json.loads(response_content)
         if not license_data.get('results', []):

src/attributecode/gen.py

@@ -281,7 +281,7 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license
 
     if gen_license:
         license_dict, err = model.pre_process_and_fetch_license_dict(
-            abouts, api_url, api_key)
+            abouts, api_url=api_url, api_key=api_key)
         if err:
             for e in err:
                 # Avoid having same error multiple times

src/attributecode/model.py

@@ -27,8 +27,10 @@
 import json
 import os
 import posixpath
+from requests import get
 import traceback
 from itertools import zip_longest
+
 import urllib
 from urllib.parse import urljoin
 from urllib.parse import urlparse
@@ -1881,17 +1883,13 @@ def detect_special_char(expression):
 
 def valid_api_url(api_url):
     try:
-        request = Request(api_url)
-        # This will always goes to exception as no key are provided.
-        # The purpose of this code is to validate the provided api_url is correct
-        urlopen(request)
-        return True
-    except HTTPError as http_e:
-        # The 403 error code is refer to "Authentication credentials were not provided.".
-        # This is correct as no key are provided.
-        if http_e.code == 403:
+        response = get(api_url)
+        # The 403 error code is expected if the api_url is pointing to DJE as no
+        # API key is provided. The 200 status code represent connection success
+        # to scancode's LicenseDB. All other exception yield to invalid api_url
+        if response.status_code == 403 or response.status_code == 200:
             return True
+        else:
+            return False
     except:
-        # All other exceptions yield to invalid api_url
-        pass
-    return False
+        return False

src/attributecode/util.py

@@ -297,13 +297,10 @@ def have_network_connection():
 
     url = "https://scancode-licensedb.aboutcode.org/"
 
-    try:
-        response = requests.get(url)
-        if response.status_code == 200:
-            return True
-        else:
-            return False
-    except requests.exceptions.RequestException:
+    response = requests.get(url)
+    if response.status_code == 200:
+        return True
+    else:
         return False
 
 

tests/test_api.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 # ============================================================================
 
+import requests
 import unittest
 from unittest import mock
 
@@ -27,6 +28,7 @@ class FakeResponse(object):
 
     def __init__(self, response_content):
         self.response_content = response_content
+        self.text = response_content
 
     def read(self):
         return self.response_content
@@ -44,12 +46,13 @@ def test_api_get_license_details_from_api(self, request_license_data):
         errors = []
         request_license_data.return_value = license_data, errors
 
-        expected = ({'short_name': 'Apache 2.0', 'full_text': 'Apache License Version 2.0 ...', 'key': 'apache-2.0'}, [])
+        expected = ({'short_name': 'Apache 2.0',
+                    'full_text': 'Apache License Version 2.0 ...', 'key': 'apache-2.0'}, [])
         result = api.get_license_details_from_api(
             api_url='api_url', api_key='api_key', license_key='license_key')
         assert expected == result
 
-    @mock.patch.object(api, 'urlopen')
+    @mock.patch.object(api, 'get')
     def test_api_request_license_data_with_result(self, mock_data):
         response_content = (
             b'{"count":1,"results":[{"name":"Apache 2.0","key":"apache-2.0","text":"Text"}]}'
@@ -63,7 +66,7 @@ def test_api_request_license_data_with_result(self, mock_data):
         )
         assert expected == license_data
 
-    @mock.patch.object(api, 'urlopen')
+    @mock.patch.object(api, 'get')
     def test_api_request_license_data_without_result(self, mock_data):
         response_content = b'{"count":0,"results":[]}'
         mock_data.return_value = FakeResponse(response_content)
@@ -79,5 +82,6 @@ def test_api_request_license_data_with_incorrect_url(self, mock_data):
         mock_data.return_value = FakeResponse(response_content)
         license_data = api.request_license_data(
             api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0')
-        expected = ({}, [Error(ERROR, "Invalid '--api_url'. License generation is skipped.")])
+        expected = (
+            {}, [Error(ERROR, "Invalid '--api_url'. License generation is skipped.")])
         assert expected == license_data