ENT-13808: Expand release-information with release files and checksums from community/releases.json

Author: sarakthon

src/cfengine_cli/masterfiles/analyze.py

@@ -93,31 +93,25 @@ def finalize_vcf(versions_dict, checksums_dict, files_dict):
     return versions_dict, checksums_dict, files_dict
 
 
-def filter_unstable_releases(data):
+def get_stable_releases(data):
     # Filter the data to only include stable releases (not debug, alpha, or beta releases):
-    filtered_data = []
-
-    for release_data in data.get("releases", []):
-        if release_data.get("debug") is True:
-            continue
-        if release_data.get("alpha") is True:
-            continue
-        if release_data.get("beta") is True:
-            continue
-
-        filtered_data.append(release_data)
-
-    return filtered_data
+    return [
+        r
+        for r in data.get("releases", [])
+        if not (r.get("debug") or r.get("alpha") or r.get("beta"))
+    ]
 
 
 def sort_release_data(file_checksums_dict):
     # Newest versions first, and files sorted alphabetically within each version
-    for v in file_checksums_dict.keys():
-        file_checksums_dict[v] = dict_sorted_by_key(file_checksums_dict[v])
+    # Work on copy to avoid mutating original dict
+    file_checksums_dict_copy = file_checksums_dict
+    for v in file_checksums_dict_copy.keys():
+        file_checksums_dict_copy[v] = dict_sorted_by_key(file_checksums_dict_copy[v])
 
     sorted_dict = OrderedDict(
         sorted(
-            file_checksums_dict.items(),
+            file_checksums_dict_copy.items(),
             key=lambda p: version_as_comparable_list(p[0]),
             reverse=True,
         )

src/cfengine_cli/masterfiles/download.py

@@ -11,7 +11,7 @@
 )
 
 ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json"
-
+COMMUNITY_RELEASES_URL = "https://cfengine.com/release-data/community/releases.json"
 
 COMMUNITY_ONLY_VERSIONS = ["3.12.0b1", "3.10.0b1"]
 """Masterfiles versions which do not appear in Enterprise releases but appear in Community releases."""

src/cfengine_cli/masterfiles/generate_release_information.py

@@ -1,9 +1,11 @@
+import json
 from cfengine_cli.masterfiles.download import (
     ENTERPRISE_RELEASES_URL,
+    COMMUNITY_RELEASES_URL,
     download_all_versions,
 )
 from cfengine_cli.masterfiles.generate_vcf_download import generate_vcf_download
-from cfengine_cli.masterfiles.analyze import filter_unstable_releases, sort_release_data
+from cfengine_cli.masterfiles.analyze import get_stable_releases, sort_release_data
 from cfengine_cli.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout
 from cfengine_cli.masterfiles.check_download_matches_git import (
     check_download_matches_git,
@@ -72,24 +74,23 @@ def generate_release_information_impl(
         )
 
 
-def generate_release_history():
-    print("Generating release history information...")
-
-    releases_data = download_releasedata()
-
-    stable_releases = filter_unstable_releases(releases_data)
-
-    file_checksums_dict = build_release_history(stable_releases)
+def download_enterprise_releasedata():
+    # Downloading releases.json:
+    try:
+        releases_data = get_json(ENTERPRISE_RELEASES_URL)
 
-    sorted_releases = sort_release_data(file_checksums_dict)
+    except CFBSNetworkError:
+        raise CFBSExitError(
+            "Downloading CFEngine release data failed - check your Wi-Fi / network settings."
+        )
 
-    write_json("./cfengine-enterprise/checksums.json", sorted_releases)
+    return releases_data
 
 
-def download_releasedata():
-    # Downloading releases.json:
+def download_community_releasedata():
+    # Downloading community/releases.json
     try:
-        releases_data = get_json(ENTERPRISE_RELEASES_URL)
+        releases_data = get_json(COMMUNITY_RELEASES_URL)
 
     except CFBSNetworkError:
         raise CFBSExitError(
@@ -99,6 +100,29 @@ def download_releasedata():
     return releases_data
 
 
+def process_release_type(folder, download_func):
+    # Function for processing either community or enterprise releases
+    release_data = download_func()
+
+    write_json_pretty(f"./{folder}/releases.json", release_data)
+
+    stable_releases = get_stable_releases(release_data)
+
+    file_checksums_dict = build_release_history(stable_releases)
+
+    write_version_files(stable_releases, folder)
+
+    sorted_releases = sort_release_data(file_checksums_dict)
+
+    write_json(f"./{folder}/checksums.json", sorted_releases)
+
+
+def generate_release_history():
+    print("Generating release history information...")
+    process_release_type("cfengine-enterprise", download_enterprise_releasedata)
+    process_release_type("cfengine-community", download_community_releasedata)
+
+
 def build_release_history(filtered_releases):
     release_history = {}
 
@@ -152,3 +176,19 @@ def extract_file_info(asset_data):
         return filename, checksum
 
     return None, None
+
+
+def write_version_files(stable_releases, folder):
+    # Writes versionfiles for each version
+    for release_data in stable_releases:
+        version = release_data.get("version")
+        if not version:
+            continue
+        version_data, _ = download_release_version_data(release_data)
+        write_json(f"./{folder}/versions/{version}.json", version_data)
+
+
+def write_json_pretty(path, data):
+    # Writes release information in same format as on cfengine.com
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)