ingest.py

"""
This is the script that gathers markdown files from all of Netdata's repos in this repo

Stages of this ingest script:

    Stage_1: Ingest every available markdown from the defaultRepos

    Stage_2: We create three buckets:
        1. all_markdown_files: all the markdown files in defaultRepos
        2. markdown_files_with_metadata: all the markdown files that have hidden metadata fields
        3. toPublish: markdown files that must be included in the learn
            (metadata_key_value: "learn_status": "Published")

    Stage_3:
        1. Move the toPublish markdown files under the DOCS_PREFIX folder based on their metadata (they decide where,
            they live)
        2. Generate autogenerated pages

    Stage_4: Sanitization
        1. Make the hidden metadata fields actual readable metadata for docusaurus

    Stage_5: Convert GH links to version specific links
"""

# Imports
import argparse
import ast
import errno
import glob
import io
import os
import re
import shutil
import urllib.parse
from pathlib import Path, PurePosixPath
import json
import xml.etree.ElementTree as ET
import git
import numpy as np
import pandas as pd
import requests
import yaml
from jsonschema import Draft7Validator

try:
    from PIL import Image
except Exception:
    Image = None

import autogenerateRedirects as genRedirects

DRY_RUN = False
DEBUG = False
DOCS_PREFIX = "will be added by arguments"

rest_files_dictionary = {}
rest_files_with_metadata_dictionary = {}
to_publish = {}
all_markdown_files = []
UNCORRELATED_LINK_COUNTER = 0
# Dict mapping repo name to dict of broken URLs -> set of source files
UNCORRELATED_URLS_BY_REPO = {}
# Dict mapping repo name to dict of (url, header) -> set of source files
BROKEN_HEADER_LINKS_BY_REPO = {}
FAIL_ON_REPOS = set()  # Set of repo names to fail on if broken links found
FAIL_ON_ALL_BROKEN_LINKS = False  # If True, fail on any broken link regardless of repo
IGNORE_ON_PREM_REPO = (
    False  # If True, skip cloning on-prem repo and ignore links pointing to it
)
LOCAL_REPOS = {}  # Dict mapping repo name to local path (use local dir instead of cloning)
# Temporarily until we release (change it (the default) to /docs
# version_prefix = "nightly"  # We use this as the version prefix in the link strategy
TEMP_FOLDER = "ingest-temp-folder"
default_repos = {
    "netdata": {
        "owner": "netdata",
        "branch": "master",
        "HEAD": "master",
    },
    "netdata-cloud-onprem": {
        "owner": "netdata",
        "branch": "master",
        "HEAD": "master",
    },
    ".github": {
        "owner": "netdata",
        "branch": "main",
        "HEAD": "main",
    },
    "agent-service-discovery": {
        "owner": "netdata",
        "branch": "master",
        "HEAD": "master",
    },
    "netdata-grafana-datasource-plugin": {
        "owner": "netdata",
        "branch": "master",
        "HEAD": "master",
    },
    "helmchart": {
        "owner": "netdata",
        "branch": "master",
        "HEAD": "master",
    },
}


# Regex to extract sidebar_position from frontmatter (supports double or single quotes)
POS_RE = re.compile(r'(?m)^\s*sidebar_position\s*:\s*["\']?(\d+)["\']?\s*$')
SIDEBAR_LABEL_RE = re.compile(r'(?m)^\s*sidebar_label\s*:\s*["\']([^"\']+)["\']\s*$')

# Marker string in auto-generated integration files
INTEGRATION_MARKER = "DO NOT EDIT THIS FILE DIRECTLY"
MAP_SIDEBAR_ORDER = {}
MAP_DOC_SCOPE = {}
MAP_SCHEMA_EXIT_CODE = 2

LOGO_ANALYSIS_TIMEOUT = 8
LOGO_RASTER_SAMPLE_SIZE = 64
LOGO_ALPHA_MIN = 16
LOGO_CONTRAST_THRESHOLD = 2.1
LIGHT_THEME_BG_RGB = (0xFD / 255.0, 0xFD / 255.0, 0xFD / 255.0)
DARK_THEME_BG_RGB = (0.0, 0.0, 0.0)
LOGO_CACHE = {}
LOGO_ANALYSIS_SUMMARY = {
    "analyzed": 0,
    "low_dark": 0,
    "low_light": 0,
    "unknown": 0,
}

MAP_COLUMNS = [
    "custom_edit_url",
    "sidebar_label",
    "learn_status",
    "learn_rel_path",
    "keywords",
    "description",
]


def format_map_schema_error(error):
    path = (
        ".".join(str(p) for p in error.absolute_path) if error.absolute_path else "root"
    )
    return f"[{path}] {error.message}"


def validate_map_schema(map_path, schema_path):
    with open(map_path, "r", encoding="utf-8") as map_file:
        map_data = yaml.safe_load(map_file) or {}

    with open(schema_path, "r", encoding="utf-8") as schema_file:
        schema = json.load(schema_file)

    validator = Draft7Validator(schema)
    errors = [
        format_map_schema_error(error) for error in validator.iter_errors(map_data)
    ]
    return len(errors) == 0, errors


def _normalize_placeholder_kind(kind):
    """Normalize integration placeholder kind to '<kind>_integrations'."""
    if not kind:
        return None
    if kind.endswith("_integrations"):
        return kind
    return f"{kind}_integrations"


def load_map_yaml(map_path):
    """Load simplified map.yaml and return ingest rows as a DataFrame.

    Rules:
    - Document rows are emitted only for nodes with non-empty `meta.edit_url`.
    - Placeholder rows are emitted for `integration_placeholder` entries.
    - `learn_rel_path` is reconstructed from hierarchy and optional `meta.path`.
    - `learn_status` is set to "Published" for emitted document rows.
    """
    with open(map_path, "r", encoding="utf-8") as fh:
        data = yaml.safe_load(fh) or {}

    sidebar = data.get("sidebar")
    if not isinstance(sidebar, list):
        raise ValueError("map.yaml must contain a top-level 'sidebar' list.")

    rows = []

    def join_path(parent_path, segment):
        if not segment:
            return parent_path
        if parent_path == "root":
            return segment
        return f"{parent_path}/{segment}"

    def walk(nodes, parent_path="root"):
        for node in nodes:
            if not node:
                continue
            if isinstance(node, dict) and node.get("type") == "integration_placeholder":
                kind = _normalize_placeholder_kind(node.get("integration_kind"))
                if kind:
                    rows.append(
                        {
                            "custom_edit_url": kind,
                            "sidebar_label": None,
                            "learn_status": None,
                            "learn_rel_path": None,
                            "keywords": None,
                            "description": None,
                        }
                    )
                continue

            if not isinstance(node, dict):
                continue

            meta = node.get("meta")
            if not isinstance(meta, dict):
                continue

            label = meta.get("label")
            has_items = isinstance(node.get("items"), list)
            explicit_path = meta.get("path")

            if has_items:
                segment = explicit_path if explicit_path else label
                current_path = join_path(parent_path, segment)
            else:
                if explicit_path:
                    current_path = join_path(parent_path, explicit_path)
                elif parent_path == "root":
                    current_path = "root"
                else:
                    current_path = parent_path

            edit_url = meta.get("edit_url")
            if isinstance(edit_url, str) and edit_url.strip():
                rows.append(
                    {
                        "custom_edit_url": edit_url,
                        "sidebar_label": label,
                        "learn_status": "Published",
                        "learn_rel_path": current_path,
                        "keywords": meta.get("keywords"),
                        "description": meta.get("description"),
                    }
                )

            if has_items:
                walk(node.get("items"), current_path)

    walk(sidebar)

    return pd.DataFrame(rows, columns=MAP_COLUMNS)


def load_map_sidebar_order(map_path):
    """Build sibling ordering map from map.yaml traversal order.

    Returns:
    - order_map: dict keyed by (parent_path, child_name) with sidebar positions
      assigned as 10, 20, 30, ... within each parent scope.
    - doc_scope_map: dict keyed by edit_url with resolved (parent_path, child_name)
      for rows originating from map.yaml documents.
    """
    with open(map_path, "r", encoding="utf-8") as fh:
        data = yaml.safe_load(fh) or {}

    sidebar = data.get("sidebar")
    if not isinstance(sidebar, list):
        return {}

    order = {}
    doc_scope = {}

    def join_path(parent_path, segment):
        if not segment:
            return parent_path
        if parent_path == "root":
            return segment
        return f"{parent_path}/{segment}"

    def walk(nodes, parent_parts):
        sibling_index = 0
        for node in nodes:
            if not isinstance(node, dict):
                continue
            if node.get("type") == "integration_placeholder":
                continue

            meta = node.get("meta") or {}
            label = meta.get("label")
            if not isinstance(label, str) or not label.strip():
                continue

            has_items = isinstance(node.get("items"), list)
            explicit_path = meta.get("path")

            if has_items:
                child_name = explicit_path if explicit_path else label
            else:
                if explicit_path:
                    child_name = explicit_path
                else:
                    child_name = label

            if not isinstance(child_name, str) or not child_name.strip():
                continue

            sibling_index += 1
            parent_key = "/".join(parent_parts) if parent_parts else "root"
            order.setdefault((parent_key, child_name), sibling_index * 10)

            edit_url = meta.get("edit_url")
            if isinstance(edit_url, str) and edit_url.strip():
                if has_items:
                    doc_scope[edit_url] = (parent_key, child_name)
                else:
                    if explicit_path:
                        leaf_parent = join_path(parent_key, child_name)
                    elif parent_key == "root":
                        leaf_parent = "root"
                    else:
                        leaf_parent = parent_key
                    doc_scope[edit_url] = (leaf_parent, label)

            if has_items:
                walk(node.get("items"), parent_parts + [child_name])

    walk(sidebar, [])
    return order, doc_scope


def ensure_category_json_for_dirs(docs_root):
    """
    For every directory under docs_root:
    - if <dir>/<basename(dir)>.mdx exists => treat as category overview; do nothing
    - else create or overwrite _category_.json using map-derived sibling order
    """
    for dirpath, dirnames, filenames in os.walk(docs_root):
        abs_dir = os.path.abspath(dirpath)
        abs_root = os.path.abspath(docs_root)

        base = os.path.basename(os.path.normpath(dirpath))

        if abs_dir == abs_root:
            continue

        category_json = os.path.join(dirpath, "_category_.json")

        if os.path.exists(os.path.join(dirpath, f"{base}.mdx")):
            continue

        mdx_files = [f for f in filenames if f.lower().endswith(".mdx")]
        if not mdx_files:
            # Do not create category files for stale empty directories.
            continue

        rel_parts = Path(dirpath).relative_to(Path(docs_root)).parts
        parent_key = "/".join(rel_parts[:-1]) if len(rel_parts) > 1 else "root"
        child_key = rel_parts[-1] if rel_parts else base

        map_pos = MAP_SIDEBAR_ORDER.get((parent_key, child_key))
        if map_pos is not None:
            cat_pos = map_pos
        else:
            # Fallback: use alphabetical-last bucket for unmapped categories
            cat_pos = 9999

        payload = {"label": base, "position": cat_pos}

        try:
            with open(category_json, "w", encoding="utf-8") as f:
                json.dump(payload, f, ensure_ascii=False, indent=2)
                f.write("\n")
        except OSError as e:
            print(f"WARNING: Failed to write category JSON '{category_json}': {e}")
            continue

        print(f"CREATE {category_json} with position={cat_pos} label='{base}'")


def _read_sidebar_label(path_to_file):
    """Read sidebar_label from frontmatter; fallback to file stem."""
    try:
        content = Path(path_to_file).read_text(encoding="utf-8")
    except OSError:
        return Path(path_to_file).stem

    match = SIDEBAR_LABEL_RE.search(content)
    if match:
        return match.group(1).strip()
    return Path(path_to_file).stem


def _set_sidebar_position(path_to_file, sidebar_position):
    """Upsert sidebar_position in frontmatter."""
    path_obj = Path(path_to_file)
    content = path_obj.read_text(encoding="utf-8")

    if POS_RE.search(content):
        updated = POS_RE.sub(
            f'sidebar_position: "{sidebar_position}"', content, count=1
        )
        path_obj.write_text(updated, encoding="utf-8")
        return

    lines = content.splitlines()
    if lines and lines[0].strip() == "---":
        lines.insert(1, f'sidebar_position: "{sidebar_position}"')
        path_obj.write_text("\n".join(lines) + "\n", encoding="utf-8")


def _set_category_position(path_to_file, label, sidebar_position):
    """Upsert position in _category_.json preserving label."""
    path_obj = Path(path_to_file)
    payload = {"label": label, "position": sidebar_position}
    path_obj.write_text(
        json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
    )


def normalize_sidebar_positions_by_parent(docs_root):
    """Assign unique sibling positions per parent scope.

    Uses map-derived sibling rank when available, then falls back to alphabetical
    ordering for injected/unknown siblings.
    """

    for dirpath, dirnames, filenames in os.walk(docs_root):
        rel_parts = Path(dirpath).relative_to(Path(docs_root)).parts
        parent_key = "/".join(rel_parts) if rel_parts else "root"

        entries = []

        # Direct child docs of this scope (exclude overview page for this scope)
        for fn in sorted(filenames):
            if not fn.lower().endswith(".mdx"):
                continue

            file_path = Path(dirpath) / fn
            if rel_parts and file_path.stem == rel_parts[-1]:
                continue

            label = _read_sidebar_label(file_path)
            entries.append(
                {
                    "path": str(file_path),
                    "child_name": label,
                    "display_label": label,
                    "kind": "doc",
                }
            )

        # Child directories represented by overview pages <child>/<child>.mdx
        for child_dir in sorted(dirnames):
            overview = Path(dirpath) / child_dir / f"{child_dir}.mdx"
            category_json = Path(dirpath) / child_dir / "_category_.json"
            if overview.exists():
                label = _read_sidebar_label(overview)
                entries.append(
                    {
                        "path": str(overview),
                        "child_name": child_dir,
                        "display_label": label,
                        "kind": "doc",
                    }
                )
            elif category_json.exists():
                try:
                    payload = json.loads(category_json.read_text(encoding="utf-8"))
                    label = str(payload.get("label") or child_dir)
                except Exception:
                    label = child_dir
                entries.append(
                    {
                        "path": str(category_json),
                        "child_name": child_dir,
                        "display_label": label,
                        "kind": "category",
                    }
                )

        if not entries:
            continue

        ordered_entries = sorted(
            entries,
            key=lambda item: (
                0
                if parent_key == "root"
                and item["display_label"].strip().lower() == "ask nedi"
                else 1,
                0
                if MAP_SIDEBAR_ORDER.get((parent_key, item["child_name"])) is not None
                else 1,
                MAP_SIDEBAR_ORDER.get((parent_key, item["child_name"]))
                if MAP_SIDEBAR_ORDER.get((parent_key, item["child_name"])) is not None
                else 0,
                item["display_label"].lower(),
            ),
        )

        for index, item in enumerate(ordered_entries, start=1):
            position_value = index * 10
            if (
                parent_key == "root"
                and item["display_label"].strip().lower() == "ask nedi"
            ):
                position_value = 0

            if item.get("kind") == "category":
                _set_category_position(
                    item["path"], item["display_label"], position_value
                )
            else:
                _set_sidebar_position(item["path"], position_value)


def clean_and_lower_string(string):
    """Normalize text for slug-like sorting and URL path segments."""
    return re.sub(
        r"(-)+",
        "-",
        string.lower().replace(",", "-").replace(" ", "-").replace("//", "/"),
    )


def extract_headers_from_file(file_path):
    """
    Extract all headers from a markdown file and return them as a set of anchor IDs.
    Headers are converted to anchor format: lowercase, spaces to hyphens, special chars removed.
    """
    headers = set()
    try:
        content = Path(file_path).read_text()
        # Match markdown headers (# Header, ## Header, etc.)
        header_pattern = r"^#{1,6}\s+(.+)$"
        for match in re.finditer(header_pattern, content, re.MULTILINE):
            header_text = match.group(1).strip()
            # Convert header to anchor ID (similar to how markdown processors do it)
            # Remove inline code backticks, bold/italic markers
            anchor = re.sub(r"[`*_]", "", header_text)
            # Remove HTML tags
            anchor = re.sub(r"<[^>]+>", "", anchor)
            # Convert to lowercase, replace spaces with hyphens
            anchor = anchor.lower().replace(" ", "-")
            # Remove special characters except hyphens
            anchor = re.sub(r"[^a-z0-9-]", "", anchor)
            # Remove multiple consecutive hyphens
            anchor = re.sub(r"-+", "-", anchor)
            # Remove leading/trailing hyphens
            anchor = anchor.strip("-")
            if anchor:
                headers.add(anchor)
    except Exception as e:
        pass
    return headers


def validate_header_in_file(file_path, header):
    """
    Check if a header/anchor exists in the target file.
    Returns True if the header exists or if header is empty, False otherwise.
    """
    if not header:
        return True
    headers = extract_headers_from_file(file_path)
    # Also check the raw header (some anchors are preserved as-is)
    return header.lower() in headers or header in headers


def extract_repo_from_github_url(url):
    """
    Extract the repository name from a GitHub URL.

    Example:
        https://github.com/netdata/netdata/blob/master/src/file.md -> netdata
        https://github.com/netdata/helmchart/blob/master/README.md -> helmchart
    """
    if not url.startswith("https://github.com/netdata/"):
        return "unknown"

    # URL format: https://github.com/netdata/<repo>/...
    parts = url.replace("https://github.com/netdata/", "").split("/")
    if parts:
        return parts[0]
    return "unknown"


def extract_repo_from_local_path(path):
    """
    Extract the repository name from a local path in the temp folder.

    Example:
        ingest-temp-folder/netdata/src/file.md -> netdata
        docs/something/file.mdx -> unknown (not from temp folder)
    """
    if path.startswith(TEMP_FOLDER + "/"):
        parts = path.replace(TEMP_FOLDER + "/", "").split("/")
        if parts:
            return parts[0]
    return "unknown"


def add_broken_url(repo, url, source_file):
    """Add a broken URL to the tracking dictionary, categorized by repo."""
    global UNCORRELATED_URLS_BY_REPO

    # If ignoring on-prem repo, skip URLs that point to it
    if IGNORE_ON_PREM_REPO:
        target_repo = extract_repo_from_github_url(url)
        if target_repo == "netdata-cloud-onprem":
            return

    if repo not in UNCORRELATED_URLS_BY_REPO:
        UNCORRELATED_URLS_BY_REPO[repo] = {}
    if url not in UNCORRELATED_URLS_BY_REPO[repo]:
        UNCORRELATED_URLS_BY_REPO[repo][url] = set()
    UNCORRELATED_URLS_BY_REPO[repo][url].add(source_file)


def add_broken_header(repo, full_link, header, source_file):
    """Add a broken header link to the tracking dictionary, categorized by repo."""
    global BROKEN_HEADER_LINKS_BY_REPO

    # If ignoring on-prem repo, skip links that point to it
    if IGNORE_ON_PREM_REPO:
        target_repo = extract_repo_from_github_url(full_link)
        if target_repo == "netdata-cloud-onprem":
            return

    if repo not in BROKEN_HEADER_LINKS_BY_REPO:
        BROKEN_HEADER_LINKS_BY_REPO[repo] = {}
    key = (full_link, header)
    if key not in BROKEN_HEADER_LINKS_BY_REPO[repo]:
        BROKEN_HEADER_LINKS_BY_REPO[repo][key] = set()
    BROKEN_HEADER_LINKS_BY_REPO[repo][key].add(source_file)


def github_url_to_local_path(url):
    """
    Convert a GitHub URL to a local path in the temp folder.
    Returns the local path if it can be constructed, None otherwise.

    Example:
        https://github.com/netdata/netdata/blob/master/src/go/pkg/prometheus/selector/README.md
        -> ingest-temp-folder/netdata/src/go/pkg/prometheus/selector/README.md
    """
    if not url.startswith("https://github.com/netdata"):
        return None

    # Convert URL to local path
    local_path = url.replace("https://github.com/netdata", TEMP_FOLDER)
    local_path = local_path.replace("edit/", "blob/", 1)
    local_path = local_path.replace("blob/master/", "")
    local_path = local_path.replace("blob/main/", "")

    return local_path


def resolve_repo_relative_link(raw_link, source_file_path):
    """
    Resolve a local markdown link into:
      1) repo-root-relative path (e.g. src/go/plugin/README.md)
      2) local temp-folder path (e.g. ingest-temp-folder/netdata/src/go/plugin/README.md)

    Returns tuple(repo_relative_path, local_path) or (None, None) if not resolvable.
    """
    if not raw_link:
        return None, None

    repo_name = extract_repo_from_local_path(source_file_path)
    if repo_name == "unknown":
        return None, None

    repo_root = Path(TEMP_FOLDER) / repo_name
    source_path = Path(source_file_path)

    try:
        source_parent = source_path.relative_to(repo_root).parent
    except ValueError:
        return None, None

    link = raw_link.replace("\\", "/").strip()

    # Root-relative links are relative to repo root. Dot-relative links are from the source file directory.
    if link.startswith("/"):
        candidate = link.lstrip("/")
    elif link.startswith("."):
        parent_prefix = source_parent.as_posix()
        candidate = f"{parent_prefix}/{link}" if parent_prefix != "." else link
    else:
        return None, None

    normalized = PurePosixPath(candidate).as_posix()
    normalized = os.path.normpath(normalized).replace("\\", "/")

    while normalized.startswith("./"):
        normalized = normalized[2:]

    if normalized in ("", ".", "..") or normalized.startswith("../"):
        return None, None

    local_path = (repo_root / Path(normalized)).as_posix()
    return normalized, local_path


def file_exists_in_repos(url):
    """
    Check if a GitHub URL points to a file that exists in the cloned repos.
    Returns True if the file exists, False otherwise.
    """
    local_path = github_url_to_local_path(url)
    if local_path is None:
        return False
    return Path(local_path).exists()


def convert_parenthetical_slash(segment: str) -> str:
    """
    Convert occurrences like "(ABC/XYZ)" into "ABC-XYZ" inside a string.
    Only converts simple parenthetical groups containing a single slash.
    """
    if not segment:
        return segment

    # Replace occurrences of (A/B) or (A/B/C) with A-B or A-B-C respectively
    def repl(m):
        inner = m.group(1)
        parts = inner.split("/")
        return "-".join(parts)

    return re.sub(r"\(([^()]+?/[^()]+?)\)", repl, segment)


def populate_integrations(markdownFiles):
    """
    Populate integration placeholders in the map using integration metadata.

    Reads integration metadata from markdown files, builds category-specific
    integration entry sets, and replaces placeholder sentinel rows in the map.
    """

    print("### Populating map from Integration metadata rows ###\n")

    metadata_dictionary = {}
    ignore_dup = []

    # Read the map file, to replace the placeholder for the dynamic part
    map_file = load_map_yaml("map.yaml")

    collectors_entries = pd.DataFrame()
    live_functions_entries = pd.DataFrame()
    exporting_entries = pd.DataFrame()
    alerting_agent_entries = pd.DataFrame()
    alerting_cloud_entries = pd.DataFrame()
    authentication_entries = pd.DataFrame()
    secretstore_entries = pd.DataFrame()
    logs_entries = pd.DataFrame()

    readmes_first = []
    others_last = []
    for file in markdownFiles:
        if "README.md" in file:
            readmes_first.append(file)
        else:
            others_last.append(file)

    markdownFiles = readmes_first + others_last

    for file in markdownFiles:
        normalized_file = file.replace("\\", "/")
        path = file.split("integrations")[0].replace("README.md", "")

        whole_file = Path(file).read_text()

        if whole_file not in ignore_dup and INTEGRATION_MARKER in whole_file:
            meta = (
                whole_file.split("endmeta-->")[0].replace("<!--startmeta", "---")
                + "---"
            )

            metadata_dictionary = read_metadata(meta)

            if os.path.islink(file):
                ignore_dup.append(whole_file)
                # If it is a manual symlink, meaning a README symlink but the folder has more than one integration, thus their custom_edit_urls are unique. 1:1 integrations have the README link as custom_edit_url
                if (
                    not file.replace("ingest-temp-folder/", "").split("/", 1)[1]
                    in metadata_dictionary["custom_edit_url"]
                ):
                    proper_edit_url = file.replace("ingest-temp-folder/", "")

                    proper_edit_url = (
                        "https://github.com/netdata/"
                        + proper_edit_url.split("/", 1)[0]
                        + "/edit/master/"
                        + proper_edit_url.split("/", 1)[1]
                    )
                    metadata_dictionary["custom_edit_url"] = proper_edit_url

                    # print("path:", file)
                    # print(metadata_dictionary)

            metadf = pd.DataFrame([metadata_dictionary])
            # print(file)
            if "/integrations/functions/" in normalized_file:
                # Compatibility for generated function tiles: ensure
                # custom_edit_url is path-based so map lookup can match files.
                proper_edit_url = file.replace("ingest-temp-folder/", "")
                if "/" in proper_edit_url:
                    repo_name, repo_rel_path = proper_edit_url.split("/", 1)
                    branch_name = "main" if repo_name == ".github" else "master"
                    metadf["custom_edit_url"] = (
                        f"https://github.com/netdata/{repo_name}/edit/{branch_name}/{repo_rel_path}"
                    )

                live_functions_entries = pd.concat([live_functions_entries, metadf])
            elif "collector" in path:
                collectors_entries = pd.concat([collectors_entries, metadf])
                # print(collectors_entries)
                # quit()
            elif "/secretstore/backends/" in normalized_file:
                secretstore_entries = pd.concat([secretstore_entries, metadf])
            elif "exporting" in path:
                exporting_entries = pd.concat([exporting_entries, metadf])
                # print(exporting_entries)
            elif "cloud-authentication" in file:
                authentication_entries = pd.concat([authentication_entries, metadf])
            # here we need a different check, as the path variable gets messed up
            elif "cloud-notifications" in file:
                # print("in")
                alerting_cloud_entries = pd.concat([alerting_cloud_entries, metadf])
            elif "logs" in file:
                # Custom location for Logs integrations, as they normally have a pretty big README that we add as a reference, as a child to the integration's folder.
                metadf["learn_rel_path"] = (
                    metadf["learn_rel_path"] + "/" + metadf["sidebar_label"]
                )

                logs_entries = pd.concat([logs_entries, metadf])
            else:
                alerting_agent_entries = pd.concat([alerting_agent_entries, metadf])

    # print("Collectors\n", collectors_entries, "Agent alerts\n", alerting_agent, "Cloud alerts\n",  alerting_cloud, "Exporting",  exporting_entries)

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "authentication_integrations"
    ].index
    # print(replace_index[0])
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            authentication_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "collectors_integrations"
    ].index
    # print(replace_index[0])
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            collectors_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "secretstore_integrations"
    ].index
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            secretstore_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "live_functions_integrations"
    ].index
    if len(replace_index) > 0:
        upper = map_file.iloc[: replace_index[0]]
        lower = map_file.iloc[replace_index[0] + 1 :]

        map_file = pd.concat(
            [
                upper,
                live_functions_entries.sort_values(
                    by=["learn_rel_path", "sidebar_label"],
                    key=lambda col: col.str.lower(),
                ),
                lower,
            ],
            ignore_index=True,
        )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "agent_notifications_integrations"
    ].index
    # print(replace_index[0])
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            alerting_agent_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "cloud_notifications_integrations"
    ].index
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            alerting_cloud_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "exporters_integrations"
    ].index
    # print(replace_index[0])
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            exporting_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    replace_index = map_file.loc[
        map_file["custom_edit_url"] == "logs_integrations"
    ].index
    upper = map_file.iloc[: replace_index[0]]
    lower = map_file.iloc[replace_index[0] + 1 :]

    map_file = pd.concat(
        [
            upper,
            logs_entries.sort_values(
                by=["learn_rel_path", "sidebar_label"], key=lambda col: col.str.lower()
            ),
            lower,
        ],
        ignore_index=True,
    )

    # Convert DataFrame to list of dicts and save as YAML
    generated_map_data = map_file.to_dict(orient="records")
    with open("ingest/generated_map.yaml", "w", encoding="utf-8") as fh:
        yaml.dump(
            generated_map_data,
            fh,
            default_flow_style=False,
            allow_unicode=True,
            sort_keys=False,
        )

    # quit()
    return map_file


def unsafe_cleanup_folders(folder_to_delete):
    """Cleanup every file in the specified folderToDelete."""
    print("Try to clean up the folder: ", folder_to_delete)
    try:
        shutil.rmtree(folder_to_delete)
        print("Done")
    except Exception as e:
        print("Couldn't delete the folder due to the exception: \n", e)


def produce_gh_view_link_for_repo(repo, file_path):
    """
    This function return the GitHub link (view link) of a repo e.g <owner>/<repo>
    Limitation it produces only  the master, main links only for the netdata org
    """
    if repo == ".github":
        return f"https://github.com/netdata/{repo}/blob/main/{file_path}"
    else:
        return f"https://github.com/netdata/{repo}/blob/master/{file_path}"


def produce_gh_edit_link_for_repo(repo, file_path):
    """
    This function return the GitHub link (edit link) of a repo e.g <owner>/<repo>
    Limitation it produces only  the master, main links only for the netdata org
    """
    if repo == ".github":
        return f"https://github.com/netdata/{repo}/edit/main/{file_path}"
    else:
        return "https://github.com/netdata/{repo}/edit/master/{file_path}"


def safe_cleanup_learn_folders(folder_to_delete):
    """
    Cleanup every file in the specified folderToDelete, that doesn't have the `part_of_learn: True`
    metadata in its metadata. It also prints the number of deleted files.
    """
    deleted_files = []
    md_files = fetch_markdown_from_repo(folder_to_delete)
    json_files = fetch_json_from_repo(folder_to_delete)
    print(
        f"Files in the {folder_to_delete} folder #{len(md_files) + len(json_files)} which are about to be deleted"
    )
    for md in md_files:
        metadata = read_metadata(Path(md).read_text().split("-->")[0])
        try:
            if "part_of_learn" in metadata.keys():
                # Reductant condition to emphasize what we are looking for when we clean up learn files
                if metadata["part_of_learn"] == "True":
                    pass
            else:
                deleted_files.append(md)
                os.remove(md)
        except Exception as e:
            print(f"Couldn't delete the {md} file reason: {e}")
    for json_file in json_files:
        deleted_files.append(json_file)
        os.remove(json_file)
    print(f"Cleaned up #{len(deleted_files)} files under {folder_to_delete} folder")


def verify_string_is_dictionary(string_input):
    """
    function to verify that a string input is of dictionary type
    """
    try:
        if isinstance(ast.literal_eval(string_input), dict):
            return True
        else:
            return False
    except:
        return False


def unpack_dictionary_string_to_dictionary(string_input):
    """Parse a dictionary-like string into a Python dictionary."""
    return ast.literal_eval(string_input)


def copy_doc(src, dest):
    """
    Copy a file
    """
    # Get the path
    try:
        shutil.copy(src, dest)
    except IOError as e:
        # ENOENT(2): file does not exist, raised also on missing dest parent dir
        if e.errno != errno.ENOENT:
            raise
        # try creating parent directories
        os.makedirs(os.path.dirname(dest))
        shutil.copy(src, dest)


def clone_repo(
    owner, repo, branch, depth, prefix_folder, gh_token=None, use_plain_https=False
):
    """
    Clone a repo using:
    - HTTPS + token (if gh_token is provided),
    - HTTPS without token (if use_plain_https=True),
    - SSH otherwise.

    After cloning, the remote URL is scrubbed to a clean HTTPS URL.
    """
    try:
        output_folder = os.path.join(prefix_folder, repo)

        if use_plain_https:
            # Forced plain HTTPS (no token)
            gh_url = f"https://github.com/{owner}/{repo}.git"
        elif gh_token:
            # HTTPS + PAT
            enc = urllib.parse.quote(gh_token, safe="")
            gh_url = f"https://x-access-token:{enc}@github.com/{owner}/{repo}.git"
        else:
            # SSH fallback
            gh_url = f"git@github.com:{owner}/{repo}.git"

        git.Git().clone(gh_url, output_folder, depth=depth, branch=branch)

        # Scrub remote to clean HTTPS
        clean_url = f"https://github.com/{owner}/{repo}.git"
        git.Git(output_folder).remote("set-url", "origin", clean_url)

        return f"Cloned the {branch} branch from {repo} repo (owner: {owner})"
    except Exception as e:
        return (
            f"Couldn't clone the {branch} branch from {repo} repo (owner: {owner}) \n"
            f" Exception {e} raised"
        )


def create_mdx_path_from_metadata(metadata):
    """
    Create a path from the documents metadata
    REQUIRED KEYS in the metadata input:
        [sidebar_label, learn_rel_path]
    In the returned (final) path we sanitize "/", "//" , "-", "," with one dash
    """
    final_file_source = metadata["sidebar_label"]
    custom_edit_url = str(metadata.get("custom_edit_url", "") or "")

    # Function docs can share identical labels (e.g. "Top Queries") across
    # many integrations. Derive filename from source function slug to avoid
    # collisions during ingest output path generation.
    if "/integrations/functions/" in custom_edit_url:
        parsed = urllib.parse.urlparse(custom_edit_url)
        function_slug = Path(parsed.path).stem
        if function_slug:
            final_file_source = function_slug.replace("-", " ")

    final_file = " ".join(
        (
            final_file_source.replace("'", " ")
            .replace(":", " ")
            .replace("/", " ")
            .replace(")", " ")
            .replace(",", " ")
            .replace("(", " ")
            .replace("`", " ")
        ).split()
    )

    slug = (
        "/{}/{}".format(metadata["learn_rel_path"], final_file)
        .lower()
        .replace(" ", "-")
        .replace("//", "/")
    )

    if slug.rsplit("/")[-1] == slug.rsplit("/")[-2]:
        return [
            "{}/{}/{}.mdx".format(
                DOCS_PREFIX, metadata["learn_rel_path"], final_file
            ).replace("//", "/"),
            re.sub(
                "//+",
                "/",
                "/{}/{}".format(metadata["learn_rel_path"], final_file)
                .lower()
                .replace(" ", "-")
                .rsplit("/", 1)[0],
            ),
        ]
    else:
        return [
            "{}/{}/{}.mdx".format(
                DOCS_PREFIX, metadata["learn_rel_path"], final_file
            ).replace("//", "/"),
            re.sub(
                "//+",
                "/",
                "/{}/{}".format(metadata["learn_rel_path"], final_file)
                .lower()
                .replace(" ", "-"),
            ),
        ]


def fetch_markdown_from_repo(output_folder):
    """Return markdown file paths recursively, including dot-directories."""
    return glob.glob(output_folder + "/**/*.md*", recursive=True) + glob.glob(
        output_folder + "/.**/*.md*", recursive=True
    )


def fetch_json_from_repo(output_folder):
    """Return JSON file paths recursively, including dot-directories."""
    return glob.glob(output_folder + "/**/*.json", recursive=True) + glob.glob(
        output_folder + "/.**/*.json", recursive=True
    )


def insert_and_read_hidden_metadata_from_doc(path_to_file, dictionary):
    """
    Taking a path of a file as input
    Identify the area with pattern " <!-- ...multiline string -->" and  converts them
    to a dictionary of key:value pairs
    """
    # TODO work here, predict yaml file from path, should be easy, if readme try os.exists for meta yaml, if inside integrations folder, try one out.

    # TODO unique in custom edit url might need custom editurl + sidebar_label so it can be reproduced here.
    repo, path = path_to_file.replace("ingest-temp-folder/", "").split("/", 1)

    if repo == ".github":
        key = "https://github.com/netdata/" + repo + "/edit/main" + "/" + path
    else:
        key = "https://github.com/netdata/" + repo + "/edit/master" + "/" + path

    output = ""
    for field in dictionary.loc[dictionary["custom_edit_url"] == key]:
        try:
            val = dictionary.loc[dictionary["custom_edit_url"] == key][field].values[0]

            # print((not val == np.nan),  val != val, val)
            val = str(val)

            if (not val == np.nan) and val != "nan":
                if field == "learn_rel_path":
                    if val == "root":
                        # print("ROOT")
                        val = "/"

                    if "Collecting Metrics" in val or "Collecting Metrics" in val:
                        output += "toc_max_heading_level: 6\n"
                        output += "toc_collapsible: true\n"

                if field == "sidebar_position":
                    output += '{0}: "{1}"\n'.format(field, val.replace('"', ""))
                else:
                    output += '{0}: "{1}"\n'.format(field, val.replace('"', ""))

        except Exception as e:
            pass

    if len(output) > 0:
        output = "<!--\n" + output + "-->\n"

        whole_file = Path(path_to_file).read_text()

        if whole_file.startswith("<!--"):
            body = whole_file.split("-->", 1)[1]
        else:
            body = whole_file

        Path(path_to_file).write_text(output + body)

        # print(path_to_file, output)
        metadata_dictionary = {}
        with open(path_to_file, "r+") as fd:
            raw_text = "".join(fd.readlines())
            pattern = r"((^<!--|^---)\n)((.|\n)*?)(\n(-->|---))"
            match_group = re.search(pattern, raw_text)
            # print(match_group)
            if match_group:
                raw_metadata = match_group[3]
                list_metadata = raw_metadata.split("\n")
                while list_metadata:
                    line = list_metadata.pop(0)
                    split_in_keywords = line.split(": ", 1)
                    key = split_in_keywords[0]
                    value = split_in_keywords[1]
                    if verify_string_is_dictionary(value):
                        value = unpack_dictionary_string_to_dictionary(value)
                    # If it's a multiline string
                    while list_metadata and len(list_metadata[0].split(": ", 1)) <= 1:
                        line = list_metadata.pop(0)
                        value = value + line.lstrip(" ")
                    value = value.strip('"')
                    metadata_dictionary[key] = value.lstrip(">-")
        # print("\n\n")
        return metadata_dictionary
    else:
        return []


def update_metadata_of_file(path_to_file, dictionary):
    """
    Taking a path of a file as input
    Identify the area with pattern
    "<!-- ...multiline string -->"
    and converts them to a dictionary
    of key:value pairs
    """

    output = ""

    for field in dictionary:
        val = dictionary[field]

        # Skip empty/null values for description and keywords to minimize diffs
        if field in ["description", "keywords"]:
            if (
                val is None
                or val == ""
                or val == "None"
                or (isinstance(val, list) and len(val) == 0)
            ):
                continue

        # Special-case keywords: accept CSV values (strings with quotes/brackets)
        # and also Python lists. Normalize to an inline YAML array.
        if field == "keywords":
            # If it's already a list, use it directly
            if isinstance(val, (list, tuple)):
                items = [
                    str(v).strip().strip('"').strip("'") for v in val if str(v).strip()
                ]
            else:
                # Normalize to string and strip outer quotes
                sval = str(val).strip()
                # Remove surrounding double quotes if present
                if sval.startswith('"') and sval.endswith('"'):
                    sval = sval[1:-1]
                if sval.startswith("'") and sval.endswith("'"):
                    sval = sval[1:-1]
                # If value is in bracket form [a,b,...], strip brackets
                if sval.startswith("[") and sval.endswith("]"):
                    sval_inner = sval[1:-1]
                else:
                    sval_inner = sval
                # Split on commas to form array items
                items = [
                    it.strip().strip('"').strip("'")
                    for it in sval_inner.split(",")
                    if it.strip()
                ]
            # Only add if we have items
            if items:
                output += f"{field}: [{', '.join(items)}]\n"
        else:
            val_str = str(val).replace('"', "")
            output += f'{field}: "{val_str}"\n'
    if len(output) > 0:
        output = "<!--\n" + output + "-->"

    whole_file = Path(path_to_file).read_text()

    if whole_file.startswith("<!--"):
        body = whole_file.split("-->", 1)[1]
    else:
        body = whole_file

    Path(path_to_file).write_text(output + body)


def read_metadata(meta):
    """Parse frontmatter-style metadata block into a dictionary."""
    metadata_dictionary = {}
    pattern = r"((<!--|---)\n)((.|\n)*?)(\n(-->|---))"
    match_group = re.search(pattern, meta)

    # If metadata is found
    if match_group:
        raw_metadata = match_group[3]
        list_metadata = raw_metadata.split("\n")
        # Split the key: value pairs
        while list_metadata:
            line = list_metadata.pop(0)
            split_in_keywords = line.split(": ", 1)
            key = split_in_keywords[0]
            value = split_in_keywords[1]
            if verify_string_is_dictionary(value):
                value = unpack_dictionary_string_to_dictionary(value)
            # If it's a multiline string
            while list_metadata and len(list_metadata[0].split(": ", 1)) <= 1:
                line = list_metadata.pop(0)
                value = value + line.lstrip(" ")
            value = value.strip('"')
            metadata_dictionary[key] = value.lstrip(">-")

    return metadata_dictionary


def _srgb_to_linear(channel):
    if channel <= 0.04045:
        return channel / 12.92
    return ((channel + 0.055) / 1.055) ** 2.4


def _relative_luminance(rgb):
    r, g, b = rgb
    rl = _srgb_to_linear(max(0.0, min(1.0, r)))
    gl = _srgb_to_linear(max(0.0, min(1.0, g)))
    bl = _srgb_to_linear(max(0.0, min(1.0, b)))
    return 0.2126 * rl + 0.7152 * gl + 0.0722 * bl


def _contrast_ratio(l1, l2):
    hi = max(l1, l2)
    lo = min(l1, l2)
    return (hi + 0.05) / (lo + 0.05)


def _blend_rgb_with_background(rgb, alpha, bg_rgb):
    a = max(0.0, min(1.0, alpha))
    return (
        rgb[0] * a + bg_rgb[0] * (1.0 - a),
        rgb[1] * a + bg_rgb[1] * (1.0 - a),
        rgb[2] * a + bg_rgb[2] * (1.0 - a),
    )


def _safe_float(value, default=0.0):
    try:
        return float(value)
    except (TypeError, ValueError):
        return default


def _parse_css_color(value):
    if not value:
        return None

    color = value.strip().strip("\"'").lower()
    color = re.sub(r"\s+", "", color)

    if color in {
        "none",
        "transparent",
        "currentcolor",
        "inherit",
        "initial",
        "unset",
        "url(#gradient)",
    }:
        return None

    named = {
        "black": (0.0, 0.0, 0.0, 1.0),
        "white": (1.0, 1.0, 1.0, 1.0),
        "gray": (0.5, 0.5, 0.5, 1.0),
        "grey": (0.5, 0.5, 0.5, 1.0),
        "silver": (0.75, 0.75, 0.75, 1.0),
    }
    if color in named:
        return named[color]

    if color.startswith("#"):
        hex_color = color[1:]
        if len(hex_color) == 3:
            r = int(hex_color[0] * 2, 16) / 255.0
            g = int(hex_color[1] * 2, 16) / 255.0
            b = int(hex_color[2] * 2, 16) / 255.0
            return (r, g, b, 1.0)
        if len(hex_color) == 4:
            r = int(hex_color[0] * 2, 16) / 255.0
            g = int(hex_color[1] * 2, 16) / 255.0
            b = int(hex_color[2] * 2, 16) / 255.0
            a = int(hex_color[3] * 2, 16) / 255.0
            return (r, g, b, a)
        if len(hex_color) == 6:
            r = int(hex_color[0:2], 16) / 255.0
            g = int(hex_color[2:4], 16) / 255.0
            b = int(hex_color[4:6], 16) / 255.0
            return (r, g, b, 1.0)
        if len(hex_color) == 8:
            r = int(hex_color[0:2], 16) / 255.0
            g = int(hex_color[2:4], 16) / 255.0
            b = int(hex_color[4:6], 16) / 255.0
            a = int(hex_color[6:8], 16) / 255.0
            return (r, g, b, a)

    rgb_match = re.match(r"rgba?\(([^)]+)\)", color)
    if rgb_match:
        parts = [part.strip() for part in rgb_match.group(1).split(",")]
        if len(parts) < 3:
            return None

        def _parse_channel(token):
            if token.endswith("%"):
                return max(0.0, min(1.0, float(token[:-1]) / 100.0))
            return max(0.0, min(1.0, float(token) / 255.0))

        try:
            r = _parse_channel(parts[0])
            g = _parse_channel(parts[1])
            b = _parse_channel(parts[2])
            a = 1.0
            if len(parts) >= 4:
                alpha_token = parts[3]
                if alpha_token.endswith("%"):
                    a = max(0.0, min(1.0, float(alpha_token[:-1]) / 100.0))
                else:
                    a = max(0.0, min(1.0, float(alpha_token)))
            return (r, g, b, a)
        except ValueError:
            return None

    return None


def _analyze_svg_luminance(content):
    try:
        root = ET.fromstring(content)
    except ET.ParseError:
        return None

    weighted_light = 0.0
    weighted_dark = 0.0
    total_weight = 0.0

    for element in root.iter():
        tag = element.tag.split("}")[-1].lower()
        if tag in {"svg", "defs", "title", "desc", "metadata", "style"}:
            continue

        style_map = {}
        style_attr = element.attrib.get("style", "")
        if style_attr:
            for declaration in style_attr.split(";"):
                if ":" in declaration:
                    key, val = declaration.split(":", 1)
                    style_map[key.strip().lower()] = val.strip()

        opacity = element.attrib.get("opacity", style_map.get("opacity", "1"))
        opacity_value = max(0.0, min(1.0, _safe_float(opacity, 1.0)))

        for paint_key, default_weight in (("fill", 1.0), ("stroke", 0.65)):
            paint = element.attrib.get(paint_key, style_map.get(paint_key, ""))
            parsed = _parse_css_color(paint)
            if not parsed:
                continue

            local_opacity_attr = f"{paint_key}-opacity"
            local_opacity = element.attrib.get(
                local_opacity_attr,
                style_map.get(local_opacity_attr, "1"),
            )
            local_opacity_value = max(0.0, min(1.0, _safe_float(local_opacity, 1.0)))

            r, g, b, color_alpha = parsed
            final_alpha = opacity_value * local_opacity_value * color_alpha
            if final_alpha <= 0.01:
                continue

            rgb = (r, g, b)
            light_luminance = _relative_luminance(
                _blend_rgb_with_background(rgb, final_alpha, LIGHT_THEME_BG_RGB)
            )
            dark_luminance = _relative_luminance(
                _blend_rgb_with_background(rgb, final_alpha, DARK_THEME_BG_RGB)
            )

            weight = max(0.05, final_alpha) * default_weight
            weighted_light += light_luminance * weight
            weighted_dark += dark_luminance * weight
            total_weight += weight

    if total_weight == 0.0:
        return None

    return {
        "light": weighted_light / total_weight,
        "dark": weighted_dark / total_weight,
        "confidence": "medium",
    }


def _analyze_raster_luminance(content):
    if Image is None:
        return None

    try:
        img = Image.open(io.BytesIO(content)).convert("RGBA")
    except Exception:
        return None

    img.thumbnail((LOGO_RASTER_SAMPLE_SIZE, LOGO_RASTER_SAMPLE_SIZE))
    pixels = img.getdata()

    weighted_light = 0.0
    weighted_dark = 0.0
    total_weight = 0.0

    for r, g, b, alpha in pixels:
        if alpha < LOGO_ALPHA_MIN:
            continue

        rgb = (r / 255.0, g / 255.0, b / 255.0)
        a = alpha / 255.0
        light_luminance = _relative_luminance(
            _blend_rgb_with_background(rgb, a, LIGHT_THEME_BG_RGB)
        )
        dark_luminance = _relative_luminance(
            _blend_rgb_with_background(rgb, a, DARK_THEME_BG_RGB)
        )

        weight = max(0.05, a)
        weighted_light += light_luminance * weight
        weighted_dark += dark_luminance * weight
        total_weight += weight

    if total_weight == 0.0:
        return None

    return {
        "light": weighted_light / total_weight,
        "dark": weighted_dark / total_weight,
        "confidence": "high",
    }


def _classify_logo_by_luminance(luminance_info):
    if not luminance_info:
        return {"light": "unknown", "dark": "unknown", "confidence": "low"}

    bg_light_l = _relative_luminance(LIGHT_THEME_BG_RGB)
    bg_dark_l = _relative_luminance(DARK_THEME_BG_RGB)
    light_contrast = _contrast_ratio(luminance_info["light"], bg_light_l)
    dark_contrast = _contrast_ratio(luminance_info["dark"], bg_dark_l)

    return {
        "light": "low" if light_contrast < LOGO_CONTRAST_THRESHOLD else "ok",
        "dark": "low" if dark_contrast < LOGO_CONTRAST_THRESHOLD else "ok",
        "confidence": luminance_info.get("confidence", "medium"),
    }


def _analyze_remote_logo(url):
    if url in LOGO_CACHE:
        return LOGO_CACHE[url]

    result = {"light": "unknown", "dark": "unknown", "confidence": "low"}

    try:
        response = requests.get(
            url,
            timeout=LOGO_ANALYSIS_TIMEOUT,
            headers={"User-Agent": "netdata-learn-ingest-logo-analyzer/1.0"},
        )
        response.raise_for_status()
        content_type = (response.headers.get("Content-Type") or "").lower()
        is_svg = url.lower().split("?", 1)[0].endswith(".svg") or "svg" in content_type

        if is_svg:
            svg_info = _analyze_svg_luminance(response.text)
            result = _classify_logo_by_luminance(svg_info)
        else:
            raster_info = _analyze_raster_luminance(response.content)
            result = _classify_logo_by_luminance(raster_info)
    except Exception:
        result = {"light": "unknown", "dark": "unknown", "confidence": "low"}

    LOGO_CACHE[url] = result
    LOGO_ANALYSIS_SUMMARY["analyzed"] += 1
    if result["light"] == "low":
        LOGO_ANALYSIS_SUMMARY["low_light"] += 1
    if result["dark"] == "low":
        LOGO_ANALYSIS_SUMMARY["low_dark"] += 1
    if result["light"] == "unknown" and result["dark"] == "unknown":
        LOGO_ANALYSIS_SUMMARY["unknown"] += 1

    return result


def _set_html_attr(tag, attr_name, attr_value):
    attr_regex = re.compile(rf'\s{re.escape(attr_name)}="[^"]*"')
    if attr_regex.search(tag):
        return attr_regex.sub(f' {attr_name}="{attr_value}"', tag, count=1)

    closing = "/>" if tag.rstrip().endswith("/>") else ">"
    insert_idx = tag.rfind(closing)
    if insert_idx == -1:
        return tag
    return tag[:insert_idx] + f' {attr_name}="{attr_value}"' + tag[insert_idx:]


def _annotate_integration_logo_tags(body):
    img_pattern = re.compile(
        r'<img\s+[^>]*src="(https?:\/\/(?:www\.)?netdata\.cloud\/img\/[^\"]+)"[^>]*\/?>',
        flags=re.IGNORECASE,
    )

    def _repl(match):
        tag = match.group(0)
        url = match.group(1)
        classification = _analyze_remote_logo(url)
        updated = _set_html_attr(tag, "data-integration-logo", "true")
        updated = _set_html_attr(
            updated, "data-logo-contrast-light", classification["light"]
        )
        updated = _set_html_attr(
            updated, "data-logo-contrast-dark", classification["dark"]
        )
        updated = _set_html_attr(
            updated, "data-logo-contrast-confidence", classification["confidence"]
        )
        return updated

    return img_pattern.sub(_repl, body)


def _escape_mdx_braces(body):
    """
    Escape bare { outside of fenced code blocks and inline code for MDX 3.

    MDX interprets {word} as a JSX expression, which breaks when the content
    is plain text from metadata (e.g. metric names like zabbix.{context}).

    This function:
    - Preserves fenced code blocks (```...```) and inline code (`...`)
    - Escapes every bare { that isn't already escaped
    - Restores style={{ which is valid JSX
    """
    preserved = []

    def _save(match):
        preserved.append(match.group(0))
        return f"\x00MDXBRACE{len(preserved) - 1}\x00"

    # Preserve fenced code blocks — must come before inline code
    body = re.sub(r"```.*?```", _save, body, flags=re.DOTALL)
    # Preserve inline code
    body = re.sub(r"`[^`\n]+`", _save, body)
    # Preserve MDX import/export statements (ESM syntax uses { for destructuring)
    body = re.sub(r"^import\s+.*$", _save, body, flags=re.MULTILINE)
    body = re.sub(
        r"^export\s+(?:default|function|const|let|var|\{).*$",
        _save,
        body,
        flags=re.MULTILINE,
    )

    # Escape every bare { not already preceded by a backslash
    body = re.sub(r"(?<!\\)\{", r"\\{", body)

    # Restore style={{ which is valid JSX (the above turns it into style=\{\{)
    body = body.replace("style=\\{\\{", "style={{")

    # Restore preserved code sections
    for i, original in enumerate(preserved):
        body = body.replace(f"\x00MDXBRACE{i}\x00", original)

    return body


def sanitize_page(path):
    """
    Converts the
        "<!--" -> "---"
        "-->" -> "---"
    It converts only the first occurrences of these patterns
    Side effect:
        If the document doesn't have purposeful metadata but it contains this pattern in it's body this function replace
        these patterns
    """

    body = Path(path).read_text()

    # Replace the metadata with comments
    body = body.replace("<!--", "---", 1)
    body = body.replace("-->", "---", 1)

    # Custom elements we don't want on GitHub
    body = body.replace("<!--unhideme", "")
    body = body.replace("unhideme-->", "")

    # MDX 3 compatibility replacements
    body = body.replace("<details><summary>", "<details>\n<summary>")
    body = body.replace("<details open><summary>", "<details open>\n<summary>")
    body = _escape_mdx_braces(body)
    if INTEGRATION_MARKER in body:
        body = _annotate_integration_logo_tags(body)
    body = body.replace("<=", r"\<=")
    body = body.replace("%<", r"%\<")
    body = body.replace("<->", r"\<->")

    # <url> into [url](url)
    body = re.sub(r"<(https://[^>]+)>", r"[\1](\1)", body)
    body = re.sub(r"<(http://[^>]+)>", r"[\1](\1)", body)
    body = re.sub(r"<([\w\.-]+@[\w\.-]+\.\w+)>", r"[\1](mailto:\1)", body)

    match_group = re.search(r'meta_yaml: "(.*)"', body)
    if match_group:
        # If the file has a meta_yaml field, then it is an integration, and we need to put the value into custom_edit_url too
        body = re.sub(
            r"meta_yaml:.*\n",
            "",
            re.sub(r"custom_edit_url:.*", f'custom_edit_url: "{match_group[1]}"', body),
        )

    # The list with the lines that will be written in the file
    output = []

    # For each line of the file I read
    for line in body.splitlines():
        # If the line isn't an H1 title, and it isn't an analytics pixel, append it to the output list
        if not line.startswith("[![analytics]"):
            output.append(line + "\n")
    output = "".join(output)
    # Try to remove excess newlines from the start of the document
    output = re.sub(r"---(\n\s*\n)", "---\n\n", output)
    # Try to add a newline to the start of a document that has no newline
    if not re.match(r"---(\n\s*\n)", output):
        # print(path, "not matching")
        output = output.replace("---\n", "---\n\n", 2)
        # revert first line
        output = output.replace("---\n\n", "---\n", 1)

    # Open the file for overwriting, we are going to write the output list in the file
    Path(path).write_text(output)


def add_new_learn_path_key_to_dict(
    input_dict, docs_prefix, docs_path_learn, temp_folder
):
    """
    This function takes as an argument our dictionary of the Ingest process and creates a new dictionary with key-value
    pairs of type Source file -> Target file (learn_absolute path)
    """
    output_dictionary = dict()
    for element in input_dict:
        repo = input_dict[element]["ingestedRepo"]
        file_path = element.replace(temp_folder + "/" + repo + "/", "")

        source_link = produce_gh_view_link_for_repo(repo, file_path)
        output_dictionary[source_link] = (
            input_dict[element]["learnPath"]
            .split(".mdx")[0]
            .lstrip('"')
            .rstrip('"')
            .replace(docs_prefix, docs_path_learn)
        )
        source_link = produce_gh_edit_link_for_repo(repo, file_path)
        output_dictionary[source_link] = (
            input_dict[element]["learnPath"]
            .split(".mdx")[0]
            .lstrip('"')
            .rstrip('"')
            .replace(docs_prefix, docs_path_learn)
        )

        # Check for pages that are category overview pages, and have filepath like ".../monitor/monitor".
        # This way we remove the double dirname in the end, because docusaurus routes the file to .../monitor
        if (
            output_dictionary[source_link].split("/")[
                len(output_dictionary[source_link].split("/")) - 1
            ]
            == output_dictionary[source_link].split("/")[
                len(output_dictionary[source_link].split("/")) - 2
            ]
        ):
            same_parent_dir = output_dictionary[source_link].split("/")[
                len(output_dictionary[source_link].split("/")) - 2
            ]

            proper_link = output_dictionary[source_link].split(same_parent_dir, 1)
            output_dictionary[source_link] = proper_link[0] + proper_link[1].strip("/")

        _temp = (
            output_dictionary[source_link]
            .replace("'", " ")
            .replace(":", " ")
            .replace(")", " ")
            .replace(",", " ")
            .replace("(", " ")
            .replace("/  +/g", " ")
            .replace(" ", "%20")
            .replace("/-+/", "-")
        )
        # If there is a slug present in the file, then that is the new_learn_path, with a "/docs" added in the front.
        try:
            input_dict[element].update(
                {"new_learn_path": "/docs" + input_dict[element]["metadata"]["slug"]}
            )
        except KeyError:
            input_dict[element].update({"new_learn_path": _temp})

    return input_dict


def local_to_absolute_links(path_to_file, input_dict):
    """
    This function is able to recognize and parse relative github links like:

    ../../../../src/database/engine/README.md
    /docs/abc.md
    """

    whole_file = Path(path_to_file).read_text()

    global UNCORRELATED_LINK_COUNTER

    # Determine the repo this file belongs to
    source_repo = extract_repo_from_local_path(path_to_file)

    # Get the custom_edit_url from input_dict for this file (the GitHub URL to the source file)
    source_github_url = path_to_file  # fallback to local path
    try:
        source_github_url = input_dict[path_to_file]["metadata"]["custom_edit_url"]
    except (KeyError, TypeError):
        pass

    # Split the file into its metadata and body, so that this function doesn't touch the metadata fields
    # metadata = "---" + whole_file.split("---", 2)[1] + "---"
    body = whole_file

    # custom_edit_url_arr = re.findall(r'custom_edit_url(.*)', metadata)

    # print(input_dict.keys())

    # If there are links inside the body
    if re.search(r"\]\((.*?)\)", body):
        # Find all the links and add them in an array
        urls_with_headers = []
        temp = re.findall(r"\[\n|.*?]\((\n|.*?)\)", body)
        # For every link, store both the base URL and the header
        for link in temp:
            base_url = link.split("#")[0]
            header = link.split("#")[1] if "#" in link else ""
            urls_with_headers.append((base_url, header, link))

        # Deduplicate by full link
        seen = set()
        unique_urls = []
        for base_url, header, full_link in urls_with_headers:
            if full_link not in seen:
                seen.add(full_link)
                unique_urls.append((base_url, header, full_link))

        for url, header, full_link in unique_urls:
            if "http" not in url and (url.startswith(".") or url.startswith("/")):
                repo_relative_path, resolved_local_path = resolve_repo_relative_link(
                    url, path_to_file
                )
                if resolved_local_path:
                    resolved_path_obj = Path(resolved_local_path)
                    if (
                        resolved_path_obj.exists()
                        and resolved_path_obj.suffix.lower() in [".md", ".mdx"]
                    ):
                        resolved_repo = extract_repo_from_local_path(path_to_file)
                        if repo_relative_path and resolved_repo != "unknown":
                            gh_view_link = produce_gh_view_link_for_repo(
                                resolved_repo, repo_relative_path
                            )
                            body = body.replace(f"({url}", "(" + gh_view_link)
                        if header and not validate_header_in_file(
                            resolved_local_path, header
                        ):
                            add_broken_header(
                                source_repo, full_link, header, source_github_url
                            )
                        continue

            # if not url.startswith("/"):

            if ".md" in url and (any(url in key for key in input_dict.keys())):
                if "http" not in url and url.startswith(".") and len(url) > 0:
                    # print("Link starts with '.'")
                    # The URL will get replaced by the value of the replaceString
                    # try:
                    url_to_replace = url
                    # print(url, path_to_file)

                    path_arr = path_to_file.split("/")

                    url_arr = url.split("/")

                    url_leftover = url_arr.copy()

                    path_arr.pop()
                    for piece in url_arr:
                        if piece == "..":
                            url_leftover.pop(0)
                            path_arr.pop()
                            print(path_arr)

                    replace = "/".join(path_arr) + "/" + "/".join(url_leftover)

                    check = Path(replace)

                    if check.exists():
                        body = body.replace(f"({url_to_replace}", "(" + replace)
                        # Validate header if present
                        if header and not validate_header_in_file(replace, header):
                            add_broken_header(
                                source_repo, full_link, header, source_github_url
                            )
                elif url.startswith("/"):
                    # print("link starting with dash")
                    url_to_replace = url

                    path_arr = path_to_file.split("/")

                    url_arr = url.split("/")

                    # print(path_arr, url_arr)

                    url_leftover = url_arr.copy()

                    replace = f"{path_arr[0]}/{path_arr[1]}" + "/".join(url_leftover)

                    check = Path(replace)

                    if check.exists():
                        body = body.replace(f"({url_to_replace}", "(" + replace)
                        # Validate header if present
                        if header and not validate_header_in_file(replace, header):
                            add_broken_header(
                                source_repo, full_link, header, source_github_url
                            )
            else:
                if url.startswith(".") or url.startswith("/"):
                    directory = "ingest-temp-folder"
                    substring = url
                    found = False

                    for root, dirs, files in os.walk(directory):
                        for name in files + dirs:
                            if substring in os.path.join(root, name):
                                # print(url, path_to_file, path_to_file.split('/')[1], "NOT IN LEARN")

                                replace = f"(https://github.com/netdata/{path_to_file.split('/')[1]}/blob/master{url}"

                                body = body.replace(f"({url}", replace)
                                found = True
                                break
                        if found:
                            break

                    if not found:
                        UNCORRELATED_LINK_COUNTER += 1
                        add_broken_url(source_repo, url, source_github_url)

    Path(path_to_file).write_text(body)


def convert_github_links(path_to_file, input_dict):
    """
    Input:
        path: The path to the markdown file
        input_dict: the dictionary with every info about all files

    Expected format of links in files:
        [*](https://github.com/netdata/netdata/blob/master/*)
        or any other Netdata repo
    """

    whole_file = Path(path_to_file).read_text()

    global UNCORRELATED_LINK_COUNTER

    # Split the file into its metadata and body, so that this function doesn't touch the metadata fields
    metadata = "---" + whole_file.split("---", 2)[1] + "---"
    body = whole_file.split("---", 2)[2]

    custom_edit_url_arr = re.findall(r"custom_edit_url(.*)", metadata)

    # Determine the source repo from the custom_edit_url in metadata
    # This tells us which repo the current file originally came from
    source_repo = "unknown"
    source_github_url = path_to_file  # fallback to local path
    if custom_edit_url_arr and len(custom_edit_url_arr[0]) > 1:
        custom_edit_url = custom_edit_url_arr[0].replace('"', "").strip(": ")
        source_repo = extract_repo_from_github_url(custom_edit_url)
        source_github_url = custom_edit_url

    # If there are links inside the body
    if re.search(r"\]\((.*?)\)", body):
        # Find all the links and add them in an array
        urls_with_headers = []
        temp = re.findall(r"\[\n|.*?]\((\n|.*?)\)", body)
        # For every link, store both the base URL and the header
        for link in temp:
            base_url = link.split("#")[0]
            header = link.split("#")[1] if "#" in link else ""
            urls_with_headers.append((base_url, header, link))

        for url, header, full_link in urls_with_headers:
            # The URL will get replaced by the value of the replaceString
            try:
                # The keys inside fileDict are like "ingest-temp-folder/netdata/collectors/charts.d.plugin/ap/README.md"
                # so from the link, we need:
                # 1. replace the https link prefix up until our organization identifier with the prefix of the temp folder
                # 2. try and catch any mishaps in links that instead of "blob" have "edit"
                # 3. remove "blob/master/" or "blob/main/"
                # 4. Then we have the correct key for the dictionary

                dict_key = (
                    url.replace("https://github.com/netdata", TEMP_FOLDER)
                    .replace("edit/", "blob/", 1)
                    .replace("blob/master/", "")
                    .replace("blob/main/", "")
                )
                dictionary = input_dict[dict_key]

                replace_string = dictionary["new_learn_path"].replace("//", "/")

                # In some cases, a "id: someId" will be in a file, this is to change a file's link in Docusaurus,
                # so we need to be careful to honor that
                try:
                    metadata_id = dictionary["metadata"]["id"]

                    replace_string = replace_string.replace(
                        replace_string.split("/")[len(replace_string.split("/")) - 1],
                        metadata_id,
                    )

                except Exception as e:
                    # There is no "id" metadata in the file, do nothing

                    pass

                body = body.replace("](" + url, "](" + replace_string)

                # Validate header if present - check against the source file in temp folder
                if header and dict_key in input_dict:
                    source_file = dict_key
                    if Path(source_file).exists() and not validate_header_in_file(
                        source_file, header
                    ):
                        # Use source_repo (where the file with the broken link is from)
                        add_broken_header(
                            source_repo, full_link, header, source_github_url
                        )

                # In the end replace the URL with the replaceString
            except Exception as e:
                # This is probably a link that can't be translated to a Learn link (e.g. An external file)
                if url.startswith("https://github.com/netdata") and re.search(
                    r"\.md", url
                ):
                    # Try to rescue an integration link
                    if "integrations" in url and ("collector" in url):
                        # Due to the integrations/cloud_notifications/integrations/.. scenario, we use rsplit to remove the last occurrence of "integrations"
                        # We want to map links to specific integrations mds, to their parent README, in case the above try-catch failed to find the replacement.
                        try_url = url.rsplit("integrations", 1)[0] + "README.md"
                        # The URL will get replaced by the value of the replaceString
                        try:
                            # The keys inside fileDict are like "ingest-temp-folder/netdata/collectors/charts.d.plugin/ap/README.md"
                            # , so from the link, we need:
                            # replace the https link prefix until our organization identifier with the prefix of the temp folder
                            # try and catch any mishaps in links that instead of "blob" have "edit"
                            # remove "blob/master/" or "blob/main/"
                            # Then we have the correct key for the dictionary

                            dict_key = (
                                try_url.replace(
                                    "https://github.com/netdata", TEMP_FOLDER
                                )
                                .replace("edit", "blob")
                                .replace("blob/master/", "")
                                .replace("blob/main/", "")
                            )
                            dictionary = input_dict[dict_key]
                            replace_string = dictionary["new_learn_path"]

                            # In some cases, a "id: someId" will be in a file, this is to change a file's link in Docusaurus,
                            # so we need to be careful to honor that
                            try:
                                metadata_id = dictionary["metadata"]["id"]

                                replace_string = replace_string.replace(
                                    replace_string.split("/")[
                                        len(replace_string.split("/")) - 1
                                    ],
                                    metadata_id,
                                )
                            except Exception as e:
                                # There is no "id" metadata in the file, do nothing
                                pass

                            # In the end replace the URL with the replaceString
                            body = body.replace("](" + url, "](" + replace_string)

                            # Validate header if present
                            if (
                                header
                                and Path(dict_key).exists()
                                and not validate_header_in_file(dict_key, header)
                            ):
                                add_broken_header(
                                    source_repo, full_link, header, source_github_url
                                )
                        except:
                            # Only mark as broken if the file doesn't exist in the repos
                            # Files that exist but aren't published to Learn are fine - they stay as GitHub links
                            if not file_exists_in_repos(url):
                                UNCORRELATED_LINK_COUNTER += 1
                                add_broken_url(source_repo, url, source_github_url)

                            if len(custom_edit_url_arr[0]) > 1:
                                custom_edit_url = (
                                    custom_edit_url_arr[0].replace('"', "").strip(":")
                                )
                            else:
                                custom_edit_url = (
                                    "NO custom_edit_url found, please add one"
                                )

                            # print(UNCORRELATED_LINK_COUNTER,
                            #       "INFO: In File:",
                            #       custom_edit_url,
                            #       "\n", "URL:", url, "\n")
                    else:
                        # Only mark as broken if the file doesn't exist in the repos
                        # Files that exist but aren't published to Learn are fine - they stay as GitHub links
                        if not file_exists_in_repos(url):
                            UNCORRELATED_LINK_COUNTER += 1
                            add_broken_url(source_repo, url, source_github_url)

    # Construct again the whole file
    whole_file = metadata + body

    # Write everything onto the file again
    Path(path_to_file).write_text(whole_file)


def automate_sidebar_position(dictionary):
    """
    Assign sidebar positions relative to each parent's siblings.

    Ordering is based on current map traversal order in the dataframe.
    Positions are assigned within each parent scope to preserve sibling order.
    """

    print("### Automating sidebar_position ###", "\n")
    positions = [-1] * len(dictionary)
    siblings_by_parent = {}

    for row_index, row in enumerate(dictionary.itertuples(index=False)):
        raw_path = getattr(row, "learn_rel_path", None)
        if raw_path is None or (isinstance(raw_path, float) and np.isnan(raw_path)):
            continue

        path = str(raw_path).strip()
        if not path or path == "nan":
            continue

        parts = [segment for segment in path.split("/") if segment]
        if not parts:
            continue

        label = str(getattr(row, "sidebar_label", "") or "").strip()

        if path == "root":
            parent_key = "root"
            child_name = label
        else:
            is_top_page = bool(parts) and label == parts[-1]
            if is_top_page:
                parent_key = "/".join(parts[:-1]) if len(parts) > 1 else "root"
                child_name = parts[-1]
            else:
                parent_key = path
                child_name = label

        map_rank = MAP_SIDEBAR_ORDER.get((parent_key, child_name))

        siblings_by_parent.setdefault(parent_key, []).append(
            {
                "row_index": row_index,
                "encounter": row_index,
                "map_rank": map_rank,
            }
        )

    for siblings in siblings_by_parent.values():
        ordered = sorted(
            siblings,
            key=lambda item: (
                0 if item["map_rank"] is not None else 1,
                item["map_rank"] if item["map_rank"] is not None else 0,
                item["encounter"],
            ),
        )
        for sibling_index, item in enumerate(ordered, start=1):
            positions[item["row_index"]] = sibling_index * 10

    return positions


def sort_files(file_array):
    """Sort integration files by maintainer type and normalized title."""
    netdata_integrations = []
    community_integrations = []

    for file in file_array:
        if Path(file).is_file():
            # [filename, filepath, banner message, banner color]
            try:
                content = Path(file).read_text(encoding="utf-8")
            except (UnicodeDecodeError, OSError):
                # Skip files that cannot be decoded or read
                continue

            # Use a normalized version of the filename as the sort key
            normalized_name = clean_and_lower_string(Path(file).stem)

            if "maintained%20by-Netdata-" in content:
                netdata_integrations.append(
                    [normalized_name, file, "by Netdata", "#00ab44"]
                )
            else:
                community_integrations.append(
                    [normalized_name, file, "by Community", "rgba(0, 0, 0, 0.25)"]
                )

    sorted_array = sorted(netdata_integrations) + sorted(community_integrations)

    return sorted_array


def get_dir_make_file_and_recurse(directory):
    """
    Recursively process a directory to generate integration grid pages.

    For each directory containing integration files (identified by INTEGRATION_MARKER),
    generates an MDX file with a grid layout displaying all integrations.

    Grid pages are created when:
    - Directory has at least one integration
    - Integrations outnumber non-integration files, or integrations are nested while
      direct files are non-integrations
    - No existing overview file exists for this directory
    """
    dir_name = os.path.dirname(directory)
    file_name = Path(directory).name
    filename = str(Path(dir_name) / file_name / f"{file_name}.mdx")

    if any(Path(directory).glob("**/*")):
        sorted_list = sort_files(Path(directory).glob("**/*"))

        sidebar_label = Path(directory).name
        dir_name = Path(directory)

        try:
            relative_dir = dir_name.relative_to(Path("docs"))
            relative_parts = [segment for segment in relative_dir.parts if segment]
        except ValueError:
            relative_parts = []

        if relative_parts:
            parent_key = (
                "/".join(relative_parts[:-1]) if len(relative_parts) > 1 else "root"
            )
            child_key = relative_parts[-1]
            sidebar_pos_value = MAP_SIDEBAR_ORDER.get((parent_key, child_key))
        else:
            sidebar_pos_value = None

        if sidebar_pos_value is not None:
            sidebar_position = f'sidebar_position: "{sidebar_pos_value}"'
        else:
            # Keep empty here; normalize_sidebar_positions_by_parent() will assign
            # deterministic sibling positions for generated overview pages.
            sidebar_position = ""

        try:
            # Compute path relative to the docs root (e.g. "docs/foo/bar" -> "foo/bar")
            relative_dir = dir_name.relative_to(Path("docs"))
            slug_path = clean_and_lower_string(str(relative_dir))
        except ValueError:
            # Fallback: use the full directory path if it is not under "docs"
            slug_path = clean_and_lower_string(str(dir_name))

        slug = "/" + slug_path.lstrip("/")

        md = f"""---
sidebar_label: "{sidebar_label}"
{sidebar_position}
hide_table_of_contents: true
learn_status: "AUTOGENERATED"
slug: "{slug}"
learn_link: "https://learn.netdata.cloud/docs/{slug_path}"
---

# {sidebar_label}

import \u007b Grid, Box \u007d from '@site/src/components/Grid_integrations';

<Grid  columns="4">
"""

        integrations = 0
        direct_integrations = 0
        direct_non_integrations = 0
        has_published_single_content = False  # Track if dir has single published integration (should not be overwritten)

        for file_array_entry in sorted_list:
            file = file_array_entry[1]
            message = file_array_entry[2]
            color = file_array_entry[3]
            if Path(file).is_file():
                try:
                    whole_file = Path(file).read_text(encoding="utf-8")
                except (UnicodeDecodeError, OSError) as e:
                    print(f"Skipping file {file} due to read/decode error: {e}")
                    continue

                direct_child = Path(file).parent == Path(directory)

                if INTEGRATION_MARKER in whole_file:
                    if direct_child:
                        direct_integrations += 1

                    meta_dict = read_metadata(whole_file)

                    # Check if this is a single published integration that should be treated as content (not grid)
                    if direct_child and direct_integrations == 1:
                        is_published = meta_dict.get("learn_status") == "Published"
                        has_custom_url = "custom_edit_url" in meta_dict
                        if is_published or has_custom_url:
                            # Count total direct files to check if this is the only one
                            direct_files = list(Path(directory).glob("*.mdx"))
                            if (
                                len(direct_files) == 1
                                and direct_files[0].stem == Path(directory).name
                            ):
                                has_published_single_content = True

                    try:
                        logo_match = re.search(
                            r'<img\s+[^>]*src="https?:\/\/(?:www\.)?netdata\.cloud\/img[^\"]*"[^>]*\/?>',
                            whole_file,
                        )
                        if not logo_match:
                            raise ValueError("No integration logo image found")

                        img = (
                            logo_match.group(0)
                            .replace(
                                'width="150"',
                                "style={{width: '90%', maxHeight: '100%', verticalAlign: 'middle' }}",
                            )
                            .replace("<img", "<img custom-image")
                        )
                    except ValueError:
                        img = ""

                    try:
                        box_to = meta_dict["learn_link"].replace(
                            "https://learn.netdata.cloud", ""
                        )
                        box_title = meta_dict["sidebar_label"]
                        md += f"""
<Box banner="{message}" banner_color="{color}" to="{box_to}" title="{box_title}">
    {img}
</Box>
"""
                        integrations += 1
                    except KeyError as e:
                        print(
                            f"Missing key {e} in grid generation logic for file {file}"
                        )
                        continue
                else:
                    if direct_child:
                        direct_non_integrations += 1
        # Create grid page if:
        # - Has integrations AND
        # - Integrations dominate (or all are in subdirs) AND
        # - No existing file would be overwritten AND
        # - Not a single published content page (those should remain as-is)
        has_nested_integrations_with_direct_docs = (
            integrations > 0
            and direct_integrations == 0
            and direct_non_integrations > 0
        )

        should_create_grid = (
            integrations > 0
            and not has_published_single_content
            and not Path(filename).is_file()
            and "docs.mdx" not in filename
            and (
                (
                    direct_non_integrations <= direct_integrations
                    and direct_integrations > 0
                )
                or (direct_non_integrations == 0 and direct_integrations == 0)
                or has_nested_integrations_with_direct_docs
            )
        )
        if should_create_grid:
            md += "\n</Grid>"
            Path(filename).parent.mkdir(parents=True, exist_ok=True)
            Path(filename).write_text(md, encoding="utf-8")

        for subdir in sorted(Path(directory).glob("*/")):
            get_dir_make_file_and_recurse(subdir)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Ingest docs from multiple repositories"
    )

    parser.add_argument(
        "--repos",
        default=[],
        nargs="+",
        help="Choose specific repo you want ingest. Format: owner/repo:branch or /path/to/local/repo or repo_name:/path/to/repo. If not set, defaults ingested.",
    )

    parser.add_argument(
        "--dry-run",
        help="Don't save a file with the output.",
        action="store_true",
    )

    parser.add_argument(
        "-d",
        "--debug",
        help="Enable debug printing",
        action="store_true",
    )

    parser.add_argument(
        "--docs-prefix",
        help="Don't save a file with the output.",
        dest="DOCS_PREFIX",
        default="docs",
    )

    parser.add_argument(
        "--fail-links",
        help="Exit with error code 1 at the end if any internal broken links are found.",
        action="store_true",
    )

    parser.add_argument(
        "--fail-links-netdata",
        help="Exit with error code 1 if broken links found in netdata repo.",
        action="store_true",
    )

    parser.add_argument(
        "--fail-links-helmchart",
        help="Exit with error code 1 if broken links found in helmchart repo.",
        action="store_true",
    )

    parser.add_argument(
        "--fail-links-onprem",
        help="Exit with error code 1 if broken links found in netdata-cloud-onprem repo.",
        action="store_true",
    )

    parser.add_argument(
        "--fail-links-asd",
        help="Exit with error code 1 if broken links found in agent-service-discovery repo.",
        action="store_true",
    )

    parser.add_argument(
        "--fail-links-grafana",
        help="Exit with error code 1 if broken links found in netdata-grafana-datasource-plugin repo.",
        action="store_true",
    )

    parser.add_argument(
        "--fail-links-github",
        help="Exit with error code 1 if broken links found in .github repo.",
        action="store_true",
    )

    parser.add_argument(
        "--gh-token",
        help="GitHub Personal Access Token for HTTPS cloning (optional, used only for local testing).",
        dest="gh_token",
        default=None,
    )

    parser.add_argument(
        "--use_plain_https",
        help="Clone using plain HTTPS without a token (overrides --gh-token if provided).",
        action="store_true",
        dest="use_plain_https",
    )

    parser.add_argument(
        "--ignore-on-prem-repo",
        help="Skip cloning the netdata-cloud-onprem repo and ignore broken links pointing to it.",
        action="store_true",
    )

    parser.add_argument(
        "--local-repo",
        dest="local_repos",
        default=[],
        nargs="+",
        help="Use a local directory instead of cloning. Format: repo_name:local_path (e.g., netdata:/path/to/netdata)",
    )

    list_of_repos_in_str = []
    # netdata/netdata:branch tkatsoulas/go.d.plugin:mybranch
    args = parser.parse_args()
    kArgs = args._get_kwargs()
    GITHUB_TOKEN = args.gh_token
    USE_PLAIN_HTTPS = args.use_plain_https

    # Create local copies from the parse_args input
    DOCS_PREFIX = args.DOCS_PREFIX
    for arg in kArgs:
        if arg[0] == "repos":
            list_of_repos_in_str = arg[1]
        if arg[0] == "dry_run":
            DRY_RUN = arg[1]
        if arg[0] == "debug" or arg[0] == "debug":
            if arg[1]:
                DEBUG = True
                print("RUNNING WITH DEBUG MESSAGES ON")
        if arg[0] == "fail_links":
            FAIL_ON_ALL_BROKEN_LINKS = arg[1]
        if arg[0] == "fail_links_netdata":
            if arg[1]:
                FAIL_ON_REPOS.add("netdata")
        if arg[0] == "fail_links_helmchart":
            if arg[1]:
                FAIL_ON_REPOS.add("helmchart")
        if arg[0] == "fail_links_onprem":
            if arg[1]:
                FAIL_ON_REPOS.add("netdata-cloud-onprem")
        if arg[0] == "fail_links_asd":
            if arg[1]:
                FAIL_ON_REPOS.add("agent-service-discovery")
        if arg[0] == "fail_links_grafana":
            if arg[1]:
                FAIL_ON_REPOS.add("netdata-grafana-datasource-plugin")
        if arg[0] == "fail_links_github":
            if arg[1]:
                FAIL_ON_REPOS.add(".github")
        if arg[0] == "ignore_on_prem_repo":
            if arg[1]:
                IGNORE_ON_PREM_REPO = True
                print(
                    "IGNORING ON-PREM REPO: Will skip cloning and ignore broken links pointing to netdata-cloud-onprem"
                )
        if arg[0] == "local_repos":
            for local_repo_str in arg[1]:
                try:
                    repo_name, local_path = local_repo_str.split(":", 1)
                    if not os.path.isdir(local_path):
                        print(
                            f"ERROR: Local path '{local_path}' for repo '{repo_name}' does not exist or is not a directory"
                        )
                        exit(-1)
                    LOCAL_REPOS[repo_name] = os.path.abspath(local_path)
                    print(
                        f"Using local directory for {repo_name}: {LOCAL_REPOS[repo_name]}"
                    )
                except ValueError:
                    print(
                        f"ERROR: Invalid --local-repo format '{local_repo_str}'. Expected format: repo_name:local_path"
                    )
                    exit(-1)

    if len(list_of_repos_in_str) > 0:
        for repo_str in list_of_repos_in_str:
            # Check if it's a local path (starts with / or contains :/)
            if repo_str.startswith("/") or ":/" in repo_str:
                # Handle local path: either /path/to/repo or repo_name:/path/to/repo
                if ":" in repo_str and not repo_str.startswith("/"):
                    # Format: repo_name:/path/to/repo
                    try:
                        repo_name, local_path = repo_str.split(":", 1)
                        if not os.path.isdir(local_path):
                            print(
                                f"ERROR: Local path '{local_path}' for repo '{repo_name}' does not exist or is not a directory"
                            )
                            exit(-1)
                        LOCAL_REPOS[repo_name] = os.path.abspath(local_path)
                        print(
                            f"Using local directory for {repo_name}: {LOCAL_REPOS[repo_name]}"
                        )
                    except ValueError:
                        print(
                            f"ERROR: Invalid format '{repo_str}'. Expected repo_name:/path/to/repo"
                        )
                        exit(-1)
                else:
                    # Format: /path/to/repo - infer repo_name from directory name or match to known repos
                    local_path = os.path.abspath(repo_str)
                    if not os.path.isdir(local_path):
                        print(
                            f"ERROR: Local path '{local_path}' does not exist or is not a directory"
                        )
                        exit(-1)
                    dir_name = os.path.basename(local_path)

                    # Try to match directory name to a known repo
                    matched_repo = None
                    if dir_name in default_repos:
                        matched_repo = dir_name
                    else:
                        # Try to find a known repo name in the directory name
                        for known_repo in default_repos.keys():
                            if (
                                known_repo.lower() in dir_name.lower()
                                or dir_name.lower() in known_repo.lower()
                            ):
                                matched_repo = known_repo
                                break

                    if not matched_repo:
                        print(
                            f"ERROR: Could not map directory '{dir_name}' to a known repository."
                        )
                        print(f"Known repos: {list(default_repos.keys())}")
                        print(
                            f"Either use explicit format 'repo_name:{local_path}' or name your directory after a known repo."
                        )
                        exit(-1)

                    LOCAL_REPOS[matched_repo] = local_path
                    print(
                        f"Using local directory for {matched_repo}: {LOCAL_REPOS[matched_repo]}"
                    )
            else:
                # Handle GitHub repo: owner/repo:branch
                repository = None
                try:
                    repo_owner, repo_with_branch = repo_str.split("/", 1)
                    repository, repo_branch = repo_with_branch.split(":", 1)
                    default_repos[repository]["owner"] = repo_owner
                    default_repos[repository]["branch"] = repo_branch
                except (TypeError, ValueError):
                    print(
                        "You specified a wrong format in at least one of the repos you want to ingest"
                    )
                    print(f"Invalid repo specification: {repo_str}")
                    parser.print_usage()
                    exit(-1)
                except KeyError:
                    print(repository or repo_str)
                    print("The repo you specified in not in predefined repos")
                    print(default_repos.keys())
                    parser.print_usage()
                    exit(-1)
                except Exception as exc:
                    print("Unknown error in parsing", exc)

    USE_PLAIN_HTTPS = USE_PLAIN_HTTPS or IGNORE_ON_PREM_REPO

    # Clean up old clones into a temp dir
    unsafe_cleanup_folders(TEMP_FOLDER)
    # Clean up old ingested docs
    safe_cleanup_learn_folders(DOCS_PREFIX)
    print("Creating a temp directory: ", TEMP_FOLDER)

    try:
        os.mkdir(TEMP_FOLDER)
    except FileExistsError:
        print("Folder already exists")

    # Clone all the predefined repos
    for repo_name in default_repos.keys():
        # Skip on-prem repo if flag is set
        if IGNORE_ON_PREM_REPO and repo_name == "netdata-cloud-onprem":
            print(f"Skipping {repo_name} (--ignore-on-prem-repo flag is set)")
            continue
        # Use local directory if specified, otherwise clone
        if repo_name in LOCAL_REPOS:
            local_path = LOCAL_REPOS[repo_name]
            dest_path = os.path.join(TEMP_FOLDER, repo_name)
            print(f"Copying local directory {local_path} to {dest_path}")
            shutil.copytree(local_path, dest_path, symlinks=True)
            print(f"Using local directory for {repo_name} (copied from {local_path})")
        else:
            print(
                clone_repo(
                    default_repos[repo_name]["owner"],
                    repo_name,
                    default_repos[repo_name]["branch"],
                    1,
                    TEMP_FOLDER + "/",
                    GITHUB_TOKEN,
                    USE_PLAIN_HTTPS,
                )
            )

    shutil.move("ingest-temp-folder/netdata/docs/.map/map.yaml", "map.yaml")

    map_schema_path = "ingest-temp-folder/netdata/docs/.map/map.schema.json"
    if not os.path.isfile(map_schema_path):
        print(f"ERROR: Required map schema not found: {map_schema_path}")
        exit(MAP_SCHEMA_EXIT_CODE)

    try:
        schema_valid, schema_errors = validate_map_schema("map.yaml", map_schema_path)
    except Exception as exc:
        print(f"ERROR: Failed to validate map.yaml schema: {exc}")
        exit(MAP_SCHEMA_EXIT_CODE)

    if not schema_valid:
        print("\n### MAP SCHEMA VALIDATION FAILED ###")
        for err in schema_errors:
            print(f"  - {err}")
        exit(MAP_SCHEMA_EXIT_CODE)

    map_sidebar_order, map_doc_scope = load_map_sidebar_order("map.yaml")
    MAP_SIDEBAR_ORDER.clear()
    MAP_SIDEBAR_ORDER.update(map_sidebar_order)
    MAP_DOC_SCOPE.clear()
    MAP_DOC_SCOPE.update(map_doc_scope)

    # We fetch the markdown files from the repositories
    all_markdown_files = fetch_markdown_from_repo(TEMP_FOLDER)
    print("Files detected: ", len(all_markdown_files), "\n")

    # Fill the mapDict with the metadata the integration mds have (autogenerated metadata)
    mapDict = populate_integrations(all_markdown_files)

    # set the index to the unique custom_edit_url column
    mapDict.set_index("custom_edit_url").T.to_dict("dict")

    # Automate the sidebar position
    mapDict["sidebar_position"] = automate_sidebar_position(mapDict)
    # Make the column type integer
    mapDict["sidebar_position"] = mapDict["sidebar_position"].astype(int)

    markdown_files_with_metadata = []

    for markdown in all_markdown_files:
        # print("File: ", markdown)
        md_metadata = insert_and_read_hidden_metadata_from_doc(markdown, mapDict)
        # Check to see if the dictionary returned is empty
        if len(md_metadata) > 0:
            # print("FOUND METADATA", markdown)
            # print(metadata)
            markdown_files_with_metadata.append(markdown)
            if (
                "learn_status" in md_metadata.keys()
                and md_metadata["learn_status"] == "Published"
            ):
                try:
                    # check the type of the response (for more info of what the response can be check
                    # the return statements of the function itself)
                    response = create_mdx_path_from_metadata(md_metadata)
                    to_publish[markdown] = {
                        "metadata": md_metadata,
                        "learnPath": str(response[0]),
                        "ingestedRepo": str(markdown.split("/", 2)[1]),
                    }

                    # print(response[1])

                    md_metadata.update(
                        {
                            "learn_link": "https://learn.netdata.cloud/docs"
                            + response[1],
                            "slug": response[1],
                        }
                    )

                    update_metadata_of_file(markdown, md_metadata)
                except KeyError as exc:
                    print(f"File {markdown} doesn't contain key-value", exc)
            else:
                # We don't need these files
                rest_files_with_metadata_dictionary[markdown] = {
                    "metadata": md_metadata,
                    "learnPath": str(f"docs/_archive/_{markdown}"),
                    "ingestedRepo": str(markdown.split("/", 2)[1]),
                }
        # Don't fail on empty markdown
        elif not os.stat(markdown).st_size == 0:
            rest_files_dictionary[markdown] = {"tmpPath": markdown}
        del md_metadata

    # Check for case-insensitive path collisions (e.g. "Power Supply.mdx" vs "Power supply.mdx").
    # These cause problems on case-insensitive filesystems (macOS, Windows) and indicate
    # duplicate sidebar_labels in upstream metadata.yaml files.
    paths_lower = {}
    for md_file, info in to_publish.items():
        lp = info["learnPath"]
        lp_lower = lp.lower()
        if lp_lower in paths_lower:
            prev_file = paths_lower[lp_lower]
            print(
                f"WARNING: Case-insensitive path collision detected!\n"
                f"  '{to_publish[prev_file]['learnPath']}' (from {prev_file})\n"
                f"  '{lp}' (from {md_file})\n"
                f"  These files will overwrite each other on case-insensitive filesystems."
            )
        else:
            paths_lower[lp_lower] = md_file
    del paths_lower

    # FILE MOVING
    print("### Moving files ###\n")

    # identify published documents
    print(f"### Found Learn files: {len(to_publish)}###\n")

    for md_file in to_publish:
        local_to_absolute_links(md_file, to_publish)
        copy_doc(md_file, to_publish[md_file]["learnPath"])
        sanitize_page(to_publish[md_file]["learnPath"])

    print("### Fixing github links ###")

    # After the moving, we have a new metadata, called new_learn_path, and we utilize that to fix links that were
    # pointing to GitHub relative paths
    file_dict = add_new_learn_path_key_to_dict(
        to_publish, DOCS_PREFIX, "/docs", TEMP_FOLDER
    )

    for md_file in to_publish:
        convert_github_links(file_dict[md_file]["learnPath"], file_dict)

    genRedirects.main(file_dict)
    print(
        "Done.",
        "Uncorrelated links (links from our github repos that the files are not in Learn):",
        UNCORRELATED_LINK_COUNTER,
    )

    # Print deduplicated list of uncorrelated URLs grouped by repo
    total_broken_urls = sum(len(urls) for urls in UNCORRELATED_URLS_BY_REPO.values())
    repos_with_broken_urls = set()
    if total_broken_urls > 0:
        print("\n### Uncorrelated URLs (grouped by repo) ###")
        for repo in sorted(UNCORRELATED_URLS_BY_REPO.keys()):
            repo_urls = UNCORRELATED_URLS_BY_REPO[repo]
            if len(repo_urls) > 0:
                repos_with_broken_urls.add(repo)
                print(f"\n=== Repo: {repo} ({len(repo_urls)} broken URLs) ===")
                for url in sorted(repo_urls.keys()):
                    print(f"\n  Broken link: {url}")
                    print(f"  Referenced in {len(repo_urls[url])} file(s):")
                    for source_file in sorted(repo_urls[url]):
                        print(f"    - {source_file}")
        print(f"\nTotal unique broken URLs: {total_broken_urls}")

    # Print broken header links grouped by repo
    total_broken_headers = sum(
        len(headers) for headers in BROKEN_HEADER_LINKS_BY_REPO.values()
    )
    repos_with_broken_headers = set()
    if total_broken_headers > 0:
        print("\n### Broken Header/Anchor Links (grouped by repo) ###")
        for repo in sorted(BROKEN_HEADER_LINKS_BY_REPO.keys()):
            repo_headers = BROKEN_HEADER_LINKS_BY_REPO[repo]
            if len(repo_headers) > 0:
                repos_with_broken_headers.add(repo)
                print(
                    f"\n=== Repo: {repo} ({len(repo_headers)} broken header links) ==="
                )
                for full_link, header in sorted(repo_headers.keys()):
                    print(f"\n  Broken link: {full_link}")
                    print(f"  Missing header: #{header}")
                    print(
                        f"  Referenced in {len(repo_headers[(full_link, header)])} file(s):"
                    )
                    for source_file in sorted(repo_headers[(full_link, header)]):
                        print(f"    - {source_file}")
        print(f"\nTotal unique broken header links: {total_broken_headers}")

    # Combine all repos with any broken links
    all_repos_with_issues = repos_with_broken_urls | repos_with_broken_headers

    # Determine if we should fail based on flags
    should_fail = False
    failed_repos = []

    # Check if --fail is set (fail on any broken link)
    if FAIL_ON_ALL_BROKEN_LINKS and len(all_repos_with_issues) > 0:
        should_fail = True
        failed_repos = list(all_repos_with_issues)

    # Check per-repo failure flags
    for repo in FAIL_ON_REPOS:
        if repo in all_repos_with_issues:
            should_fail = True
            if repo not in failed_repos:
                failed_repos.append(repo)

    # Store failure state but continue processing
    SHOULD_EXIT_WITH_FAILURE = False
    if should_fail:
        SHOULD_EXIT_WITH_FAILURE = True
        print(
            f"\n### BROKEN LINKS DETECTED in repos: {', '.join(sorted(failed_repos))} ###"
        )
        print("Will exit with error code 1 at the end due to flags:")
        if FAIL_ON_ALL_BROKEN_LINKS:
            print("  --fail-links")
        for repo in sorted(failed_repos):
            if repo in FAIL_ON_REPOS:
                # Map repo names to flag names
                flag_map = {
                    "netdata": "--fail-links-netdata",
                    "helmchart": "--fail-links-helmchart",
                    "netdata-cloud-onprem": "--fail-links-onprem",
                    "agent-service-discovery": "--fail-links-asd",
                    "netdata-grafana-datasource-plugin": "--fail-links-grafana",
                    ".github": "--fail-links-github",
                }
                print(f"  {flag_map.get(repo, repo)}")
        print("\nContinuing with remaining ingest steps...")

    if DEBUG:
        # Print the list of markdown not in Learn, for debugging purposes
        if len(rest_files_dictionary):
            print("ABORT: Files found that are not in the map, exiting...")
            for md_file in rest_files_dictionary:
                print(rest_files_dictionary[md_file]["tmpPath"])

    # Write the current dict into a file, so we can check for redirects in the next commit
    temp_dict = {}
    custom_edit_urls_array = []
    new_learn_paths_array = []

    for repo_name in file_dict:
        custom_edit_urls_array.append(
            file_dict[repo_name]["metadata"]["custom_edit_url"]
        )
        new_learn_paths_array.append(file_dict[repo_name]["new_learn_path"])

    temp_dict["custom_edit_url"] = custom_edit_urls_array
    temp_dict["learn_path"] = new_learn_paths_array

    df = pd.DataFrame.from_dict(temp_dict)
    df.set_index("custom_edit_url")
    # Convert DataFrame to list of dicts and save as YAML
    redirect_data = df.to_dict(orient="records")
    with open("./ingest/one_commit_back_file-dict.yaml", "w", encoding="utf-8") as fh:
        yaml.dump(
            redirect_data,
            fh,
            default_flow_style=False,
            allow_unicode=True,
            sort_keys=False,
        )

    unsafe_cleanup_folders(TEMP_FOLDER)
    os.remove("map.yaml")

    # check if we need integration grid
    get_dir_make_file_and_recurse(f"./{DOCS_PREFIX}")
    # After needed grid pages are made, check if there are leftover dirs without overview pages
    ensure_category_json_for_dirs(DOCS_PREFIX)
    # Normalize sibling sidebar positions (docs + _category_.json) to avoid UI ordering collisions
    normalize_sidebar_positions_by_parent(DOCS_PREFIX)

    if LOGO_ANALYSIS_SUMMARY["analyzed"] > 0:
        print("\n### Integration logo contrast analysis ###")
        print(f"Analyzed logos: {LOGO_ANALYSIS_SUMMARY['analyzed']}")
        print(f"Low contrast in dark theme: {LOGO_ANALYSIS_SUMMARY['low_dark']}")
        print(f"Low contrast in light theme: {LOGO_ANALYSIS_SUMMARY['low_light']}")
        print(f"Unknown classifications: {LOGO_ANALYSIS_SUMMARY['unknown']}")

    # Print final operation status with exit code
    if SHOULD_EXIT_WITH_FAILURE:
        print("\n" + "=" * 60)
        print("OPERATION FINISHED - FAILURE (exit code: 1)")
        print("Broken links detected matching failure criteria")
        print("=" * 60)
    else:
        print("\n" + "=" * 60)
        print("OPERATION FINISHED - SUCCESS (exit code: 0)")
        print("=" * 60)

    # Exit with failure if broken links were detected and failure flag was set
    if SHOULD_EXIT_WITH_FAILURE:
        exit(1)