Fixed #534 - fixed licenses group issue

 * update changelog and reference.rst

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

CHANGELOG.rst

@@ -2,13 +2,16 @@
 Changelog
 
 2023-xx-xx
-    Release 10.0.1
+    Release 10.1.0
 
     * Fixed `transform` with nested list #531
     * Added curl dependency in Dockerfile #532
     * Introduce spdx_license_expression
     * Ability to transform spdx license key from spdx_license_expression to
-      license_expression
+      license_expression (i.e. Generate attribution with
+      spdx_license_expression) #513
+    * Ability to configure the proxy settings #533
+    * Fixed licenses issue #534
 
 2023-08-20
     Release 10.0.0

docs/source/reference.rst

@@ -83,8 +83,8 @@ Options
 Purpose
 -------
 
-Generate an attribution file which contains license information
-from the INPUT along with the license text.
+Generate an attribution file which contains license information from the INPUT
+along with the license text.
 
 Assume the following:
 
@@ -421,6 +421,60 @@ Details
                     This option tells the tool to show all errors found.
                     The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING'
 
+Special Notes
+-------------
+If the input contains values for license_file, the tool will attempt to
+associate the license_file with the corresponding license_key.
+
+sample.csv
+
++----------------+------+---------------------+--------------+
+| about_resource | name | license_expression  | license_file |
++================+======+=====================+==============+
+| /project/test.c| test.c | mit AND custom    | custom.txt   |
++----------------+------+---------------------+--------------+
+
+If the user does not utilize the **--fetch-license** option, the input will
+contain two license keys and one license file. In this scenario, the tool cannot
+determine which license key the license file is referencing. As a result, the
+license_file will be saved separately.
+
+i.e.
+
+        ..  code-block:: none
+
+                about_resource: test.c
+                name: test.c
+                license_expression: mit AND custom
+                licenses:
+                  - key: mit
+                    name: mit
+                  - key: custom
+                    name: custom
+                  - file: custom.txt
+
+On the other hand, if the user generates ABOUT files using the
+**--fetch-license** option, the MIT license will be retrieved. This will result
+in having one license key and one license file. In such cases, the tool will
+consider it a successful match.
+
+i.e.
+
+        ..  code-block:: none
+
+                about_resource: test.c
+                name: test.c
+                license_expression: mit AND custom
+                licenses:
+                  - key: mit
+                    name: MIT License
+                    file: mit.LICENSE
+                    url: https://scancode-licensedb.aboutcode.org/mit.LICENSE
+                    spdx_license_key: MIT
+                  - key: custom
+                    name: custom
+                    file: custom.txt
+
 gen_license
 ===========
 
@@ -780,3 +834,20 @@ version 32.0.0 or later. If you are using an earlier version of Scancode Toolkit
 specifically version 31 or older, it will only be compatible with prior versions
 of AboutCode Toolkit.
 
+
+Configure proxy
+---------------
+The `requests` library is used since AboutCode Toolkit version 10.1.0. To do the
+http request, users can set the standard environment variables **http_proxy**,
+**https_proxy**, **no_proxy**, **all_proxy** with the export statement
+
+i.e.
+
+        ..  code-block:: none
+
+                $ export HTTP_PROXY="http://10.10.1.10:3128"
+                $ export HTTPS_PROXY="http://10.10.1.10:1080"
+                $ export ALL_PROXY="socks5://10.10.1.10:3434"
+
+See https://requests.readthedocs.io/en/latest/user/advanced/#proxies for
+references

src/attributecode/attrib.py

@@ -323,7 +323,7 @@ def generate_and_save(abouts, is_about_input, license_dict, output_location, sca
     )
 
     if rendering_error:
-        errors.extend(rendering_error)
+        errors.append(rendering_error)
 
     if rendered:
         output_location = add_unc(output_location)

src/attributecode/model.py

@@ -1224,6 +1224,13 @@ def dumps(self, licenses_dict=None):
             else:
                 if field.value:
                     data[field.name] = field.value
+        # If there is no license_key value, parse the license_expression
+        # and get the parsed license key
+        if 'license_expression' in data:
+            if not license_key and data['license_expression']:
+                _spec_char, lic_list = parse_license_expression(
+                    data['license_expression'])
+                license_key = lic_list
 
         # Group the same license information in a list
         # This `licenses_dict` is a dictionary with license key as the key and the
@@ -1246,20 +1253,35 @@ def dumps(self, licenses_dict=None):
                     lic_dict['spdx_license_key'] = spdx_lic_key
 
                 # Remove the license information if it has been handled
-                lic_key_copy.remove(lic_key)
-                if lic_name in license_name:
-                    license_name.remove(lic_name)
-                if lic_url in license_url:
-                    license_url.remove(lic_url)
-                if lic_filename in license_file:
-                    license_file.remove(lic_filename)
-                if spdx_lic_key in spdx_license_key:
-                    spdx_license_key.remove(spdx_lic_key)
-                lic_dict_list.append(lic_dict)
+                # The following condition is to check if license information
+                # has been fetched, the license key is invalid or custom if
+                # no value for lic_name
+                if lic_name:
+                    lic_key_copy.remove(lic_key)
+                    if lic_name in license_name:
+                        license_name.remove(lic_name)
+                    if lic_url in license_url:
+                        license_url.remove(lic_url)
+                    if lic_filename in license_file:
+                        license_file.remove(lic_filename)
+                    if spdx_lic_key in spdx_license_key:
+                        spdx_license_key.remove(spdx_lic_key)
+                    lic_dict_list.append(lic_dict)
 
         # Handle license information that have not been handled.
-        license_group = list(zip_longest(
-            lic_key_copy, license_name, license_file, license_url, spdx_license_key))
+        # If the len of the lic_key is the same as the lic_file, the tool should
+        # assume the lic_file (custom license) is referring this specific lic_key
+        # otherwise, the tool shouldn't group them
+        if len(lic_key_copy) == len(license_file):
+            license_group = list(zip_longest(
+                lic_key_copy, license_name, license_file, license_url, spdx_license_key))
+        else:
+            license_group = list(zip_longest(
+                lic_key_copy, license_name, [], license_url, spdx_license_key))
+            # Add the unhandled_lic_file if any
+            if license_file:
+                for lic_file in license_file:
+                    license_group.append((None, None, lic_file, None, None))
 
         for lic_group in license_group:
             lic_dict = {}
@@ -1280,15 +1302,15 @@ def dumps(self, licenses_dict=None):
             lic_dict_list.append(lic_dict)
 
         # Format the license information in the same order of the license expression
-        if license_key:
-            for key in license_key:
-                for lic_dict in lic_dict_list:
-                    if key == lic_dict['key']:
-                        data.setdefault('licenses', []).append(lic_dict)
-                        break
-        else:
+        for key in license_key:
             for lic_dict in lic_dict_list:
-                data.setdefault('licenses', []).append(lic_dict)
+                if key == lic_dict['key']:
+                    data.setdefault('licenses', []).append(lic_dict)
+                    lic_dict_list.remove(lic_dict)
+                    break
+
+        for lic_dict in lic_dict_list:
+            data.setdefault('licenses', []).append(lic_dict)
 
         return saneyaml.dump(data)