Exclude some additional URLs from check

Author: Denis-Averin

scripts/check-urls.py

@@ -45,6 +45,10 @@ class Curl:
     "https://dashboard.aspose.cloud/applications": (Curl.HTTP_RETURNED_ERROR, 404),
 }
 
+REGEX_TO_IGNORE: list[re.Pattern] = [
+    re.compile(r"^https://github\.com/(?P<user>[^/]+)/(?P<repo>[^/]+)/(?:blob|issues|pull)/.+$"),
+]
+
 URLS_TO_IGNORE: frozenset[str] = frozenset(
     [
         "https://api.aspose.cloud",
@@ -62,6 +66,7 @@ class Curl:
         ".dartlang.org",
         ".getcomposer.org",
         ".go.dev",
+        ".golang.org",
         ".google.com",
         ".gradle.org",
         ".ietf.org",
@@ -83,19 +88,21 @@ class Curl:
         ".sonatype.org",
         ".w3.org",
         ".wikipedia.org",
+        # Regular domains
+        "editorconfig.org",
     ]
 )
 
 URL_END_CHARS = r",#\)\"'<>\*\s\\"
 URL_RE_PATTERN = r"(https*://[^{0}]+)[{0}]?".format(URL_END_CHARS)
 # print(URL_RE_PATTERN)
-URL_REGEX = re.compile(URL_RE_PATTERN, re.MULTILINE)
+EXTRACT_URL_REGEX = re.compile(URL_RE_PATTERN, re.MULTILINE)
 
 # URL : [Files]
 EXTRACTED_URLS_WITH_FILES: dict[str, list[str]] = {k: [] for k in URLS_TO_IGNORE}
 
 
-def valid_url(url: str) -> bool:
+def should_check_url(url: str) -> bool:
     try:
         parsed: urllib.parse.ParseResult = urllib.parse.urlparse(url)
     except:
@@ -113,12 +120,17 @@ def valid_url(url: str) -> bool:
         # Ignore templates with {{var}}
         return False
 
+    for r in REGEX_TO_IGNORE:
+        if r.match(url):
+            # print("Ignore by regex", r.pattern, ":", url, file=sys.stderr)
+            return False
+
     return True
 
 
 def url_extractor(text: str, filename: str) -> typing.Generator[str, None, None]:
-    for url in URL_REGEX.findall(text):
-        if not valid_url(url):
+    for url in EXTRACT_URL_REGEX.findall(text):
+        if not should_check_url(url):
             # print("Ignore:", url)
             continue
         if url not in EXTRACTED_URLS_WITH_FILES:

scripts/subdomains.py

@@ -11,17 +11,17 @@ def __init__(self, domains: typing.Sequence[str]):
         self.plain_domains = set()
 
         tmp_level_with_dom: defaultdict[int, list[tuple[str, ...]]] = collections.defaultdict(list)
-        for d in domains:
+        for d in map(self.normalize_domain, domains):
             if d.startswith("."):
                 level, parts = self.get_level(d)
                 tmp_level_with_dom[level].append(parts)
             else:
                 self.plain_domains.add(d)
-
         # Ensure sorted by level
         self.domains_by_levels = tuple((key, tmp_level_with_dom[key]) for key in sorted(tmp_level_with_dom.keys()))
 
     def exists(self, domain_name: str) -> bool:
+        domain_name = self.normalize_domain(domain_name)
         if domain_name in self.plain_domains:
             return True
 
@@ -48,11 +48,16 @@ def get_level(domain_name: str) -> tuple[int, tuple[str, ...]]:
         parts = domain_name.strip(".").split(".")
         return len(parts), tuple(reversed(parts))
 
+    @staticmethod
+    def normalize_domain(domain_name: str) -> str:
+        return domain_name.lower()
+
 
 def test() -> None:
-    sd = Subdomains([".very.long.domain.name", "android.com", ".google.com"])
+    sd = Subdomains([".very.long.domain.name", "android.com", ".google.com", "editorconfig.org"])
     assert sd.exists("test.google.com")
     assert not sd.exists("test.android.com")
+    assert sd.exists("EditorConfig.org")
 
 
 if __name__ == "__main__":