Merge remote-tracking branch 'origin/master' into rust-filesystem-storage

Author: janbuchar

.github/workflows/manual_release_stable.yaml

@@ -157,12 +157,13 @@ jobs:
           rm -rf "versioned_docs/version-${MAJOR_MINOR}"
           rm -rf "versioned_sidebars/version-${MAJOR_MINOR}-sidebars.json"
           jq 'map(select(. != env.MAJOR_MINOR))' versions.json > tmp.json && mv tmp.json versions.json
-          # Copy changelog
-          cp ../CHANGELOG.md ../docs/changelog.md
           # Build API reference and create version snapshots
           bash build_api_reference.sh
           npx docusaurus docs:version "$MAJOR_MINOR"
           npx docusaurus api:version "$MAJOR_MINOR"
+          # Changelog is not versioned - it is copied from root at build time
+          rm -f "versioned_docs/version-${MAJOR_MINOR}/changelog.md"
+          echo "changelog.md" > "versioned_docs/version-${MAJOR_MINOR}/.gitignore"
 
       - name: Commit and push versioned docs
         id: commit_versioned_docs

CHANGELOG.md

@@ -2,15 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
-<!-- git-cliff-unreleased-start -->
-## 1.6.2 - **not yet released**
+## [1.6.2](https://github.com/apify/crawlee-python/releases/tag/v1.6.2) (2026-04-08)
 
 ### 🐛 Bug Fixes
 
 - **file-system:** Reclaim orphaned in-progress requests on RQ recovery ([#1825](https://github.com/apify/crawlee-python/pull/1825)) ([e86794a](https://github.com/apify/crawlee-python/commit/e86794a6e5605432c9331c7cd99edf885527a3eb)) by [@vdusek](https://github.com/vdusek)
+- Prevent premature `EventManager` shutdown when multiple crawlers share it ([#1810](https://github.com/apify/crawlee-python/pull/1810)) ([2efb668](https://github.com/apify/crawlee-python/commit/2efb668ad54fb3e8d740066446563d1e8a39d2e8)) by [@Mantisus](https://github.com/Mantisus), closes [#1805](https://github.com/apify/crawlee-python/issues/1805), [#1808](https://github.com/apify/crawlee-python/issues/1808)
+- Apply SQLite optimizations to the custom `connection_string` in `SqlStorageClient` ([#1837](https://github.com/apify/crawlee-python/pull/1837)) ([8b53e27](https://github.com/apify/crawlee-python/commit/8b53e273067e27b4ef4b2b4bb40277b15ef6b058)) by [@Mantisus](https://github.com/Mantisus)
+- Apply `SharedTimeout` to post-navigation hooks ([#1839](https://github.com/apify/crawlee-python/pull/1839)) ([88bd05a](https://github.com/apify/crawlee-python/commit/88bd05a2127ebfe3cd4eb78c514a63fc9e2cd079)) by [@vdusek](https://github.com/vdusek)
 
 
-<!-- git-cliff-unreleased-end -->
 ## [1.6.1](https://github.com/apify/crawlee-python/releases/tag/v1.6.1) (2026-03-30)
 
 ### 🐛 Bug Fixes

pyproject.toml

@@ -273,6 +273,10 @@ exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]
 [tool.ipdb]
 context = 7
 
+[tool.uv]
+# Minimal defense against supply-chain attacks.
+exclude-newer = "24 hours"
+
 # Run tasks with: uv run poe <task>
 [tool.poe.tasks]
 clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache .uv-cache build coverage-unit.xml dist htmlcov website/.docusaurus website/.yarn website/module_shortcuts.json website/node_modules "

renovate.json

@@ -21,6 +21,7 @@
             "automergeType": "branch"
         }
     ],
+    "minimumReleaseAge": "1 day",
     "schedule": ["before 7am every weekday"],
     "ignoreDeps": ["crawlee", "docusaurus-plugin-typedoc-api"]
 }

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -119,6 +119,7 @@ def _create_static_content_crawler_pipeline(self) -> ContextPipeline[ParsedHttpC
         """Create static content crawler context pipeline with expected pipeline steps."""
         return (
             ContextPipeline()
+            .compose(self._manage_shared_navigation_timeout)
             .compose(self._execute_pre_navigation_hooks)
             .compose(self._make_http_request)
             .compose(self._execute_post_navigation_hooks)
@@ -127,26 +128,37 @@ def _create_static_content_crawler_pipeline(self) -> ContextPipeline[ParsedHttpC
             .compose(self._handle_blocked_request_by_content)
         )
 
-    async def _execute_pre_navigation_hooks(
+    async def _manage_shared_navigation_timeout(
         self, context: BasicCrawlingContext
     ) -> AsyncGenerator[BasicCrawlingContext, None]:
-        context_id = id(context)
-        self._shared_navigation_timeouts[context_id] = SharedTimeout(self._navigation_timeout)
+        """Initialize and clean up the shared navigation timeout for the current request."""
+        request_id = id(context.request)
+        self._shared_navigation_timeouts[request_id] = SharedTimeout(self._navigation_timeout)
 
         try:
-            for hook in self._pre_navigation_hooks:
-                async with self._shared_navigation_timeouts[context_id]:
-                    await hook(context)
-
             yield context
         finally:
-            self._shared_navigation_timeouts.pop(context_id, None)
+            self._shared_navigation_timeouts.pop(request_id, None)
+
+    async def _execute_pre_navigation_hooks(
+        self, context: BasicCrawlingContext
+    ) -> AsyncGenerator[BasicCrawlingContext, None]:
+        request_id = id(context.request)
+
+        for hook in self._pre_navigation_hooks:
+            async with self._shared_navigation_timeouts[request_id]:
+                await hook(context)
+
+        yield context
 
     async def _execute_post_navigation_hooks(
         self, context: HttpCrawlingContext
     ) -> AsyncGenerator[HttpCrawlingContext, None]:
+        request_id = id(context.request)
+
         for hook in self._post_navigation_hooks:
-            await hook(context)
+            async with self._shared_navigation_timeouts[request_id]:
+                await hook(context)
 
         yield context
 
@@ -262,7 +274,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
         Yields:
             The original crawling context enhanced by HTTP response.
         """
-        async with self._shared_navigation_timeouts[id(context)] as remaining_timeout:
+        async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:
             result = await self._http_client.crawl(
                 request=context.request,
                 session=context.session,

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -768,27 +768,36 @@ def sigint_handler() -> None:
         return final_statistics
 
     async def _run_crawler(self) -> None:
-        event_manager = self._service_locator.get_event_manager()
+        local_event_manager = self._service_locator.get_event_manager()
+        global_event_manager = service_locator.get_event_manager()
+        if local_event_manager is global_event_manager:
+            local_event_manager = None  # Avoid entering the same event manager context twice
+
+        # The event managers are always entered.
+        contexts_to_enter: list[Any] = (
+            [global_event_manager, local_event_manager] if local_event_manager else [global_event_manager]
+        )
 
         # Collect the context managers to be entered. Context managers that are already active are excluded,
         # as they were likely entered by the caller, who will also be responsible for exiting them.
-        contexts_to_enter = [
-            cm
-            for cm in (
-                event_manager,
-                self._snapshotter,
-                self._statistics,
-                self._session_pool if self._use_session_pool else None,
-                self._http_client,
-                self._crawler_state_rec_task,
-                *self._additional_context_managers,
-            )
-            if cm and getattr(cm, 'active', False) is False
-        ]
+        contexts_to_enter.extend(
+            [
+                cm
+                for cm in (
+                    self._snapshotter,
+                    self._statistics,
+                    self._session_pool if self._use_session_pool else None,
+                    self._http_client,
+                    self._crawler_state_rec_task,
+                    *self._additional_context_managers,
+                )
+                if cm and getattr(cm, 'active', False) is False
+            ]
+        )
 
         async with AsyncExitStack() as exit_stack:
             for context in contexts_to_enter:
-                await exit_stack.enter_async_context(context)  # ty: ignore[invalid-argument-type]
+                await exit_stack.enter_async_context(context)
 
             await self._autoscaled_pool.run()
 

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -193,6 +193,7 @@ def __init__(
         # Compose the context pipeline with the Playwright-specific context enhancer.
         kwargs['_context_pipeline'] = (
             ContextPipeline()
+            .compose(self._manage_shared_navigation_timeout)
             .compose(self._open_page)
             .compose(self._navigate)
             .compose(self._execute_post_navigation_hooks)
@@ -216,6 +217,18 @@ def __init__(
 
         super().__init__(**kwargs)
 
+    async def _manage_shared_navigation_timeout(
+        self, context: BasicCrawlingContext
+    ) -> AsyncGenerator[BasicCrawlingContext, None]:
+        """Initialize and clean up the shared navigation timeout for the current request."""
+        request_id = id(context.request)
+        self._shared_navigation_timeouts[request_id] = SharedTimeout(self._navigation_timeout)
+
+        try:
+            yield context
+        finally:
+            self._shared_navigation_timeouts.pop(request_id, None)
+
     async def _open_page(
         self,
         context: BasicCrawlingContext,
@@ -242,21 +255,17 @@ async def _open_page(
             goto_options=GotoOptions(**self._goto_options),
         )
 
-        context_id = id(pre_navigation_context)
-        self._shared_navigation_timeouts[context_id] = SharedTimeout(self._navigation_timeout)
+        request_id = id(pre_navigation_context.request)
 
-        try:
-            # Only use the page context manager here — it sets the current page in a context variable,
-            # making it accessible to PlaywrightHttpClient in subsequent pipeline steps.
-            async with browser_page_context(crawlee_page.page):
-                for hook in self._pre_navigation_hooks:
-                    async with self._shared_navigation_timeouts[context_id]:
-                        await hook(pre_navigation_context)
-
-                # Yield should be inside the browser_page_context.
-                yield pre_navigation_context
-        finally:
-            self._shared_navigation_timeouts.pop(context_id, None)
+        # Only use the page context manager here — it sets the current page in a context variable,
+        # making it accessible to PlaywrightHttpClient in subsequent pipeline steps.
+        async with browser_page_context(crawlee_page.page):
+            for hook in self._pre_navigation_hooks:
+                async with self._shared_navigation_timeouts[request_id]:
+                    await hook(pre_navigation_context)
+
+            # Yield should be inside the browser_page_context.
+            yield pre_navigation_context
 
     def _prepare_request_interceptor(
         self,
@@ -329,7 +338,7 @@ async def _navigate(
             await context.page.route(context.request.url, route_handler)
 
         try:
-            async with self._shared_navigation_timeouts[id(context)] as remaining_timeout:
+            async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:
                 response = await context.page.goto(
                     context.request.url, timeout=remaining_timeout.total_seconds() * 1000, **context.goto_options
                 )
@@ -496,8 +505,12 @@ async def _handle_blocked_request_by_content(
     async def _execute_post_navigation_hooks(
         self, context: PlaywrightPostNavCrawlingContext
     ) -> AsyncGenerator[PlaywrightPostNavCrawlingContext, None]:
+        request_id = id(context.request)
+
         for hook in self._post_navigation_hooks:
-            await hook(context)
+            async with self._shared_navigation_timeouts[request_id]:
+                await hook(context)
+
         yield context
 
     async def _create_crawling_context(

src/crawlee/events/_event_manager.py

@@ -93,24 +93,20 @@ def __init__(
             delay=self._persist_state_interval,
         )
 
-        # Flag to indicate the context state.
-        self._active = False
+        # Reference count for active contexts.
+        self._active_ref_count = 0
 
     @property
     def active(self) -> bool:
         """Indicate whether the context is active."""
-        return self._active
+        return self._active_ref_count > 0
 
     async def __aenter__(self) -> EventManager:
-        """Initialize the event manager upon entering the async context.
+        """Initialize the event manager upon entering the async context."""
+        self._active_ref_count += 1
+        if self._active_ref_count > 1:
+            return self
 
-        Raises:
-            RuntimeError: If the context manager is already active.
-        """
-        if self._active:
-            raise RuntimeError(f'The {self.__class__.__name__} is already active.')
-
-        self._active = True
         self._emit_persist_state_event_rec_task.start()
         return self
 
@@ -127,17 +123,24 @@ async def __aexit__(
         Raises:
             RuntimeError: If the context manager is not active.
         """
-        if not self._active:
+        if not self.active:
             raise RuntimeError(f'The {self.__class__.__name__} is not active.')
 
+        if self._active_ref_count > 1:
+            # Emit persist state event to ensure the latest state is saved before closing the context.
+            await self._emit_persist_state_event()
+            await self.wait_for_all_listeners_to_complete(timeout=self._close_timeout)
+            self._active_ref_count -= 1
+            return
+
         # Stop persist state event periodic emission and manually emit last one to ensure latest state is saved.
         await self._emit_persist_state_event_rec_task.stop()
         await self._emit_persist_state_event()
         await self.wait_for_all_listeners_to_complete(timeout=self._close_timeout)
         self._event_emitter.remove_all_listeners()
         self._listener_tasks.clear()
         self._listeners_to_wrappers.clear()
-        self._active = False
+        self._active_ref_count -= 1
 
     @overload
     def on(self, *, event: Literal[Event.PERSIST_STATE], listener: EventListener[EventPersistStateData]) -> None: ...

src/crawlee/events/_local_event_manager.py

@@ -75,7 +75,10 @@ async def __aenter__(self) -> LocalEventManager:
         It starts emitting system info events at regular intervals.
         """
         await super().__aenter__()
-        self._emit_system_info_event_rec_task.start()
+
+        if self._active_ref_count == 1:
+            self._emit_system_info_event_rec_task.start()
+
         return self
 
     async def __aexit__(
@@ -88,7 +91,9 @@ async def __aexit__(
 
         It stops emitting system info events and closes the event manager.
         """
-        await self._emit_system_info_event_rec_task.stop()
+        if self._active_ref_count == 1:
+            await self._emit_system_info_event_rec_task.stop()
+
         await super().__aexit__(exc_type, exc_value, exc_traceback)
 
     async def _emit_system_info_event(self) -> None: