Merge pull request #709 from pmattsso/urlutils

Added helper to extract filenames from url

Author: mergify[bot]

src/fromager/commands/build.py

@@ -34,6 +34,7 @@
     wheels,
 )
 
+from .. import resolver
 from ..log import VERBOSE_LOG_FMT, ThreadLogFilter, req_ctxvar_context
 
 logger = logging.getLogger(__name__)
@@ -477,7 +478,7 @@ def _is_wheel_built(
         pbi = wkctx.package_build_info(req)
         build_tag_from_settings = pbi.build_tag(resolved_version)
         build_tag = build_tag_from_settings if build_tag_from_settings else (0, "")
-        wheel_basename = urlparse(url).path.rsplit("/", 1)[-1]
+        wheel_basename = resolver.extract_filename_from_url(url)
         _, _, build_tag_from_name, _ = parse_wheel_filename(wheel_basename)
         existing_build_tag = build_tag_from_name if build_tag_from_name else (0, "")
         if (

src/fromager/resolver.py

@@ -103,6 +103,13 @@ def default_resolver_provider(
     )
 
 
+def extract_filename_from_url(url: str) -> str:
+    """Extract filename from URL and decode it."""
+    path = urlparse(url).path
+    filename = os.path.basename(path)
+    return unquote(filename)
+
+
 class LogReporter(resolvelib.BaseReporter):
     """Report resolution events
 
@@ -186,9 +193,8 @@ def get_project_from_pypi(
         )
         # PEP 592: Check if package was yanked
         reason_data_yanked = i.attrib.get("data-yanked")
-        path = urlparse(candidate_url).path
         # file names are URL quoted, "1.0%2Blocal" -> "1.0+local"
-        filename = unquote(path.rsplit("/", 1)[-1])
+        filename = extract_filename_from_url(candidate_url)
         found_candidates.add(filename)
         if DEBUG_RESOLVER:
             logger.debug("%s: candidate %r -> %r", project, candidate_url, filename)

src/fromager/sources.py

@@ -3,13 +3,11 @@
 import inspect
 import json
 import logging
-import os.path
 import pathlib
 import shutil
 import tarfile
 import typing
 import zipfile
-from urllib.parse import unquote, urlparse
 
 import resolvelib
 from packaging.requirements import Requirement
@@ -272,7 +270,7 @@ def download_url(
     basename = (
         destination_filename
         if destination_filename
-        else unquote(os.path.basename(urlparse(url).path))
+        else resolver.extract_filename_from_url(url)
     )
     outfile = pathlib.Path(destination_dir) / basename
     logger.debug(

src/fromager/wheels.py

@@ -10,7 +10,6 @@
 import textwrap
 import typing
 import zipfile
-from urllib.parse import unquote, urlparse
 
 import elfdeps
 import tomlkit
@@ -372,9 +371,7 @@ def download_wheel(
     wheel_url: str,
     output_directory: pathlib.Path,
 ) -> pathlib.Path:
-    wheel_filename = output_directory / unquote(
-        os.path.basename(urlparse(wheel_url).path)
-    )
+    wheel_filename = output_directory / resolver.extract_filename_from_url(wheel_url)
     if not wheel_filename.exists():
         logger.info(f"downloading pre-built wheel {wheel_url}")
         wheel_filename = _download_wheel_check(req, output_directory, wheel_url)

tests/test_resolver.py

@@ -910,3 +910,18 @@ def test_pep592_support_constraint_mismatch():
 
         with pytest.raises(resolvelib.resolvers.ResolverException):
             rslvr.resolve([Requirement("setuptools-scm")])
+
+
+@pytest.mark.parametrize(
+    "url,filename",
+    [
+        ("http://example.com/path/to/file.txt", "file.txt"),
+        (
+            "http://localhost:8080/simple/vllm/vllm-0.10.0%2Brhai1-15-cp312-cp312-linux_x86_64.whl",
+            "vllm-0.10.0+rhai1-15-cp312-cp312-linux_x86_64.whl",
+        ),
+    ],
+)
+def test_extract_filename_from_url(url, filename):
+    result = resolver.extract_filename_from_url(url)
+    assert result == filename