fix: fix convert relative link to absolute in enqueue_links for response with redirect (#956)

Mantisus · web-flow · commit 694102e163bb · 2025-02-04T09:09:09.000+01:00
### Description - fix `enqueue_links` for response with redirect. ### Issues - Closes: #955
diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
@@ -152,7 +152,8 @@ async def enqueue_links(
             for link in self._parser.find_links(parsed_content, selector=selector):
                 url = link
                 if not is_url_absolute(url):
-                    url = convert_to_absolute_url(context.request.url, url)
+                    base_url = context.request.loaded_url or context.request.url
+                    url = convert_to_absolute_url(base_url, url)
 
                 request_options = RequestOptions(url=url, user_data={**base_user_data}, label=label)
 
diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
@@ -210,7 +210,8 @@ async def enqueue_links(
                         url = url.strip()
 
                         if not is_url_absolute(url):
-                            url = convert_to_absolute_url(context.request.url, url)
+                            base_url = context.request.loaded_url or context.request.url
+                            url = convert_to_absolute_url(base_url, url)
 
                         request_option = RequestOptions({'url': url, 'user_data': {**base_user_data}, 'label': label})
 
diff --git a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py
@@ -19,6 +19,8 @@
 @pytest.fixture
 async def server() -> AsyncGenerator[respx.MockRouter, None]:
     with respx.mock(base_url='https://test.io', assert_all_called=False) as mock:
+        mock.get('https://www.test.io/').return_value = Response(302, headers={'Location': 'https://test.io/'})
+
         mock.get('/', name='index_endpoint').return_value = Response(
             200,
             text="""<html>
@@ -104,14 +106,15 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
         visit(context.request.url)
         await context.enqueue_links()
 
-    await crawler.run(['https://test.io/'])
+    await crawler.run(['https://www.test.io/'])
 
     assert server['index_endpoint'].called
     assert server['secondary_index_endpoint'].called
 
     visited = {call[0][0] for call in visit.call_args_list}
+
     assert visited == {
-        'https://test.io/',
+        'https://www.test.io/',
         'https://test.io/asdf',
         'https://test.io/hjkl',
         'https://test.io/qwer',
diff --git a/tests/unit/crawlers/_parsel/test_parsel_crawler.py b/tests/unit/crawlers/_parsel/test_parsel_crawler.py
@@ -21,6 +21,8 @@
 @pytest.fixture
 async def server() -> AsyncGenerator[respx.MockRouter, None]:
     with respx.mock(base_url='https://test.io', assert_all_called=False) as mock:
+        mock.get('https://www.test.io/').return_value = Response(302, headers={'Location': 'https://test.io/'})
+
         mock.get('/', name='index_endpoint').return_value = Response(
             200,
             text="""<html>
@@ -134,14 +136,14 @@ async def request_handler(context: ParselCrawlingContext) -> None:
         visit(url)
         await context.enqueue_links()
 
-    await crawler.run(['https://test.io/'])
+    await crawler.run(['https://www.test.io/'])
 
     assert server['index_endpoint'].called
     assert server['secondary_index_endpoint'].called
 
     visited = {call[0][0] for call in visit.call_args_list}
     assert visited == {
-        'https://test.io/',
+        'https://www.test.io/',
         'https://test.io/asdf',
         'https://test.io/hjkl',
         'https://test.io/qwer',
diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
@@ -48,8 +48,9 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
 
 
 async def test_enqueue_links() -> None:
-    requests = ['https://crawlee.dev/docs/examples']
-    crawler = PlaywrightCrawler()
+    # www.crawlee.dev create a redirect to crawlee.dev
+    requests = ['https://www.crawlee.dev/docs/examples']
+    crawler = PlaywrightCrawler(max_requests_per_crawl=11)
     visit = mock.Mock()
 
     @crawler.router.default_handler
@@ -59,9 +60,13 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
 
     await crawler.run(requests)
 
-    visited: set[str] = {call[0][0] for call in visit.call_args_list}
+    first_visited = visit.call_args_list[0][0][0]
+    visited: set[str] = {call[0][0] for call in visit.call_args_list[1:]}
 
+    # The first link visited use original domain
+    assert first_visited == 'https://www.crawlee.dev/docs/examples'
     assert len(visited) >= 10
+    # All other links must have a domain name after the redirect
     assert all(url.startswith('https://crawlee.dev/docs/examples') for url in visited)
 
 
